test/CodeGen/AMDGPU/dpp_combine_rev_opcode.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu8.02 < %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -mtriple=amdgpu10.30 < %s | FileCheck -check-prefix=GFX1030 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
; RUN: llc -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
; RUN: llc -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s

; End to end coverage for the opcodes whose operands the sequencer reverses at
; runtime, so that DPP is applied to src1 instead of src0. Folding a DPP mov
; into src0 of one of those, either directly or by commuting a use into one, has
; to be rejected.

; We run the tests with and without True16, as DPP is currently not implemented for
; VGPR16 operands. As a result, the True16-enabled runs will not try to fold into
; the DPP at all on some test cases

; The DPP value is src1 of the subtract, so folding it would require commuting
; into v_subrev_u32/v_sub_nc_u32's REV form. Must stay a separate v_mov_b32_dpp.
; On gfx8 the subtract carries, so this is v_subrev_u32's carry out form.
define amdgpu_kernel void @dpp_sub_u32_rev(ptr addrspace(1) %arg, i32 %in) {
; GFX8-LABEL: dpp_sub_u32_rev:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_dword v2, v[0:1]
; GFX8-NEXT:    v_mov_b32_e32 v3, s2
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_mov_b32_dpp v3, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v3
; GFX8-NEXT:    flat_store_dword v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_sub_u32_rev:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX9-NEXT:    v_mov_b32_e32 v2, s2
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v2
; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_sub_u32_rev:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX10-NEXT:    v_mov_b32_e32 v2, s2
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v1, v2
; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_sub_u32_rev:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_clause 0x1
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX1030-NEXT:    v_mov_b32_e32 v2, s2
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    v_sub_nc_u32_e32 v1, v1, v2
; GFX1030-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-LABEL: dpp_sub_u32_rev:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_clause 0x1
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX11-NEXT:    v_mov_b32_e32 v2, s2
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_sub_nc_u32_e32 v1, v1, v2
; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: dpp_sub_u32_rev:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT:    v_mov_b32_e32 v2, s2
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_sub_nc_u32_e32 v1, v1, v2
; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds i32, ptr addrspace(1) %arg, i32 %id
  %load = load i32, ptr addrspace(1) %gep
  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in, i32 %load, i32 1, i32 15, i32 15, i1 true)
  %sub = sub i32 %load, %dpp
  store i32 %sub, ptr addrspace(1) %gep
  ret void
}

; The DPP value is already src0, so no commute is needed and the fold is legal.
define amdgpu_kernel void @dpp_sub_u32(ptr addrspace(1) %arg, i32 %in) {
; GFX8-LABEL: dpp_sub_u32:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_dword v2, v[0:1]
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_sub_u32_dpp v2, vcc, v2, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    flat_store_dword v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_sub_u32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_sub_u32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_sub_u32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_sub_nc_u32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_sub_u32:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_sub_nc_u32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-LABEL: dpp_sub_u32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_sub_nc_u32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: dpp_sub_u32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_sub_nc_u32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds i32, ptr addrspace(1) %arg, i32 %id
  %load = load i32, ptr addrspace(1) %gep
  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in, i32 %load, i32 1, i32 15, i32 15, i1 true)
  %sub = sub i32 %dpp, %load
  store i32 %sub, ptr addrspace(1) %gep
  ret void
}

; Same for the 16 bit subtract, which is v_subrev_u16 on gfx9.
define amdgpu_kernel void @dpp_sub_u16_rev(ptr addrspace(1) %arg, i16 %in) {
; GFX8-LABEL: dpp_sub_u16_rev:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
; GFX8-NEXT:    v_mov_b32_e32 v3, s2
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_mov_b32_dpp v3, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    v_sub_u16_e32 v2, v2, v3
; GFX8-NEXT:    flat_store_short v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_sub_u16_rev:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX9-NEXT:    v_mov_b32_e32 v2, s2
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    v_sub_u16_e32 v1, v1, v2
; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_sub_u16_rev:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX10-NEXT:    v_mov_b32_e32 v2, s2
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    v_sub_nc_u16 v1, v1, v2
; GFX10-NEXT:    global_store_short v0, v1, s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_sub_u16_rev:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_clause 0x1
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX1030-NEXT:    v_mov_b32_e32 v2, s2
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    v_sub_nc_u16 v1, v1, v2
; GFX1030-NEXT:    global_store_short v0, v1, s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-TRUE16-LABEL: dpp_sub_u16_rev:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_clause 0x1
; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x2c
; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_lshlrev_b32 v1, 1, v0
; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1]
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-TRUE16-NEXT:    v_sub_nc_u16 v0.l, v0.l, v2.l
; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
; GFX11-TRUE16-NEXT:    s_endpgm
;
; GFX11-FAKE16-LABEL: dpp_sub_u16_rev:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_clause 0x1
; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-FAKE16-NEXT:    s_load_b32 s2, s[4:5], 0x2c
; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1]
; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, s2
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT:    v_sub_nc_u16 v1, v1, v2
; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
; GFX11-FAKE16-NEXT:    s_endpgm
;
; GFX12-TRUE16-LABEL: dpp_sub_u16_rev:
; GFX12-TRUE16:       ; %bb.0:
; GFX12-TRUE16-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_lshlrev_b32 v1, 1, v0
; GFX12-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1]
; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-TRUE16-NEXT:    v_sub_nc_u16 v0.l, v0.l, v2.l
; GFX12-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
; GFX12-TRUE16-NEXT:    s_endpgm
;
; GFX12-FAKE16-LABEL: dpp_sub_u16_rev:
; GFX12-FAKE16:       ; %bb.0:
; GFX12-FAKE16-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1]
; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v2, s2
; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT:    v_sub_nc_u16 v1, v1, v2
; GFX12-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
; GFX12-FAKE16-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds i16, ptr addrspace(1) %arg, i32 %id
  %load = load i16, ptr addrspace(1) %gep
  %dpp = call i16 @llvm.amdgcn.update.dpp.i16(i16 %in, i16 %load, i32 1, i32 15, i32 15, i1 true)
  %sub = sub i16 %load, %dpp
  store i16 %sub, ptr addrspace(1) %gep
  ret void
}

; The 64 bit subtract covers the carry out and carry in halves, which would
; commute into v_subrev_co_u32 and v_subbrev_u32 (v_subrev_co_ci_u32 on gfx10+).
define amdgpu_kernel void @dpp_sub_i64_rev(ptr addrspace(1) %arg, i64 %in) {
; GFX8-LABEL: dpp_sub_i64_rev:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT:    v_mov_b32_e32 v5, s2
; GFX8-NEXT:    v_mov_b32_e32 v4, s3
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    v_mov_b32_dpp v5, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    v_mov_b32_dpp v4, v3 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    v_sub_u32_e32 v2, vcc, v2, v5
; GFX8-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_sub_i64_rev:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[0:1]
; GFX9-NEXT:    v_mov_b32_e32 v4, s2
; GFX9-NEXT:    v_mov_b32_e32 v3, s3
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_mov_b32_dpp v4, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    v_mov_b32_dpp v3, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v4
; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_sub_i64_rev:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_dwordx2 v[0:1], v2, s[0:1]
; GFX10-NEXT:    v_mov_b32_e32 v3, s2
; GFX10-NEXT:    v_mov_b32_e32 v4, s3
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_mov_b32_dpp v3, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    v_mov_b32_dpp v4, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, v3
; GFX10-NEXT:    v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo
; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_sub_i64_rev:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1030-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_dwordx2 v[0:1], v2, s[0:1]
; GFX1030-NEXT:    v_mov_b32_e32 v3, s2
; GFX1030-NEXT:    v_mov_b32_e32 v4, s3
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_mov_b32_dpp v3, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    v_mov_b32_dpp v4, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, v3
; GFX1030-NEXT:    v_sub_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
; GFX1030-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-LABEL: dpp_sub_i64_rev:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_dual_mov_b32 v3, s2 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
; GFX11-NEXT:    v_mov_b32_e32 v4, s3
; GFX11-NEXT:    global_load_b64 v[0:1], v2, s[0:1]
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_mov_b32_dpp v3, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-NEXT:    v_mov_b32_dpp v4, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, v3
; GFX11-NEXT:    v_sub_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: dpp_sub_i64_rev:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    v_dual_mov_b32 v3, s2 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
; GFX12-NEXT:    v_mov_b32_e32 v4, s3
; GFX12-NEXT:    global_load_b64 v[0:1], v2, s[0:1]
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_mov_b32_dpp v3, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-NEXT:    v_mov_b32_dpp v4, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, v3
; GFX12-NEXT:    v_sub_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds i64, ptr addrspace(1) %arg, i32 %id
  %load = load i64, ptr addrspace(1) %gep
  %dpp = call i64 @llvm.amdgcn.update.dpp.i64(i64 %in, i64 %load, i32 1, i32 15, i32 15, i1 true)
  %sub = sub i64 %load, %dpp
  store i64 %sub, ptr addrspace(1) %gep
  ret void
}

; Float subtracts are implemented as an add with a negate modifier rather than
; with reversed operands, so v_subrev_f32 still takes the DPP fold.
define amdgpu_kernel void @dpp_fsub_f32(ptr addrspace(1) %arg, float %in) {
; GFX8-LABEL: dpp_fsub_f32:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_dword v2, v[0:1]
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_subrev_f32_dpp v2, v2, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    flat_store_dword v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_fsub_f32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_subrev_f32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_fsub_f32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_subrev_f32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_fsub_f32:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_subrev_f32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-LABEL: dpp_fsub_f32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_subrev_f32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: dpp_fsub_f32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_subrev_f32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds float, ptr addrspace(1) %arg, i32 %id
  %load = load float, ptr addrspace(1) %gep
  %dpp = call float @llvm.amdgcn.update.dpp.f32(float %in, float %load, i32 1, i32 15, i32 15, i1 true)
  %sub = fsub float %load, %dpp
  store float %sub, ptr addrspace(1) %gep
  ret void
}

; Same for v_subrev_f16.
define amdgpu_kernel void @dpp_fsub_f16(ptr addrspace(1) %arg, half %in) {
; GFX8-LABEL: dpp_fsub_f16:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_subrev_f16_dpp v2, v2, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    flat_store_short v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_fsub_f16:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_subrev_f16_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_fsub_f16:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_subrev_f16_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    global_store_short v0, v1, s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_fsub_f16:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_subrev_f16_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    global_store_short v0, v1, s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-TRUE16-LABEL: dpp_fsub_f16:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_clause 0x1
; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x2c
; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_lshlrev_b32 v1, 1, v0
; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1]
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v0.l, v0.l, v2.l
; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
; GFX11-TRUE16-NEXT:    s_endpgm
;
; GFX11-FAKE16-LABEL: dpp_fsub_f16:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1]
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT:    v_subrev_f16_e64_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
; GFX11-FAKE16-NEXT:    s_endpgm
;
; GFX12-TRUE16-LABEL: dpp_fsub_f16:
; GFX12-TRUE16:       ; %bb.0:
; GFX12-TRUE16-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_lshlrev_b32 v1, 1, v0
; GFX12-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1]
; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v0.l, v0.l, v2.l
; GFX12-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
; GFX12-TRUE16-NEXT:    s_endpgm
;
; GFX12-FAKE16-LABEL: dpp_fsub_f16:
; GFX12-FAKE16:       ; %bb.0:
; GFX12-FAKE16-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1]
; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT:    v_subrev_f16_e64_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
; GFX12-FAKE16-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds half, ptr addrspace(1) %arg, i32 %id
  %load = load half, ptr addrspace(1) %gep
  %dpp = call half @llvm.amdgcn.update.dpp.f16(half %in, half %load, i32 1, i32 15, i32 15, i1 true)
  %sub = fsub half %load, %dpp
  store half %sub, ptr addrspace(1) %gep
  ret void
}

; The shift amount is src0 of v_lshlrev_b32, so the DPP mov would be folded
; straight into the operand the sequencer swaps away. Rejected before gfx11,
; where the shift REV opcodes stopped reversing their operands.
define amdgpu_kernel void @dpp_shl_amount(ptr addrspace(1) %arg, i32 %in) {
; GFX8-LABEL: dpp_shl_amount:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_dword v2, v[0:1]
; GFX8-NEXT:    v_mov_b32_e32 v3, s2
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_mov_b32_dpp v3, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    v_lshlrev_b32_e32 v2, v3, v2
; GFX8-NEXT:    flat_store_dword v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_shl_amount:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX9-NEXT:    v_mov_b32_e32 v2, s2
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    v_lshlrev_b32_e32 v1, v2, v1
; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_shl_amount:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX10-NEXT:    v_mov_b32_e32 v2, s2
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    v_lshlrev_b32_e32 v1, v2, v1
; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_shl_amount:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_clause 0x1
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX1030-NEXT:    v_mov_b32_e32 v2, s2
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    v_lshlrev_b32_e32 v1, v2, v1
; GFX1030-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-LABEL: dpp_shl_amount:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_lshlrev_b32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: dpp_shl_amount:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_lshlrev_b32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds i32, ptr addrspace(1) %arg, i32 %id
  %load = load i32, ptr addrspace(1) %gep
  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in, i32 %load, i32 1, i32 15, i32 15, i1 true)
  %shl = shl i32 %load, %dpp
  store i32 %shl, ptr addrspace(1) %gep
  ret void
}

; Same for v_lshrrev_b32.
define amdgpu_kernel void @dpp_lshr_amount(ptr addrspace(1) %arg, i32 %in) {
; GFX8-LABEL: dpp_lshr_amount:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_dword v2, v[0:1]
; GFX8-NEXT:    v_mov_b32_e32 v3, s2
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_mov_b32_dpp v3, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    v_lshrrev_b32_e32 v2, v3, v2
; GFX8-NEXT:    flat_store_dword v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_lshr_amount:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX9-NEXT:    v_mov_b32_e32 v2, s2
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_lshr_amount:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX10-NEXT:    v_mov_b32_e32 v2, s2
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_lshr_amount:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_clause 0x1
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX1030-NEXT:    v_mov_b32_e32 v2, s2
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    v_lshrrev_b32_e32 v1, v2, v1
; GFX1030-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-LABEL: dpp_lshr_amount:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_lshrrev_b32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: dpp_lshr_amount:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_lshrrev_b32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds i32, ptr addrspace(1) %arg, i32 %id
  %load = load i32, ptr addrspace(1) %gep
  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in, i32 %load, i32 1, i32 15, i32 15, i1 true)
  %shr = lshr i32 %load, %dpp
  store i32 %shr, ptr addrspace(1) %gep
  ret void
}

; Same for v_ashrrev_i32.
define amdgpu_kernel void @dpp_ashr_amount(ptr addrspace(1) %arg, i32 %in) {
; GFX8-LABEL: dpp_ashr_amount:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_dword v2, v[0:1]
; GFX8-NEXT:    v_mov_b32_e32 v3, s2
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_mov_b32_dpp v3, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    v_ashrrev_i32_e32 v2, v3, v2
; GFX8-NEXT:    flat_store_dword v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_ashr_amount:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX9-NEXT:    v_mov_b32_e32 v2, s2
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    v_ashrrev_i32_e32 v1, v2, v1
; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_ashr_amount:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX10-NEXT:    v_mov_b32_e32 v2, s2
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    v_ashrrev_i32_e32 v1, v2, v1
; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_ashr_amount:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_clause 0x1
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX1030-NEXT:    v_mov_b32_e32 v2, s2
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    v_ashrrev_i32_e32 v1, v2, v1
; GFX1030-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-LABEL: dpp_ashr_amount:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_ashrrev_i32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: dpp_ashr_amount:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_ashrrev_i32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds i32, ptr addrspace(1) %arg, i32 %id
  %load = load i32, ptr addrspace(1) %gep
  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in, i32 %load, i32 1, i32 15, i32 15, i1 true)
  %shr = ashr i32 %load, %dpp
  store i32 %shr, ptr addrspace(1) %gep
  ret void
}

; Same for the 16 bit shift, v_lshlrev_b16.
define amdgpu_kernel void @dpp_shl_i16_amount(ptr addrspace(1) %arg, i16 %in) {
; GFX8-LABEL: dpp_shl_i16_amount:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
; GFX8-NEXT:    v_mov_b32_e32 v3, s2
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_mov_b32_dpp v3, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    v_lshlrev_b16_e32 v2, v3, v2
; GFX8-NEXT:    flat_store_short v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_shl_i16_amount:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX9-NEXT:    v_mov_b32_e32 v2, s2
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    v_lshlrev_b16_e32 v1, v2, v1
; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_shl_i16_amount:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX10-NEXT:    v_mov_b32_e32 v2, s2
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    v_lshlrev_b16 v1, v2, v1
; GFX10-NEXT:    global_store_short v0, v1, s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_shl_i16_amount:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_clause 0x1
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX1030-NEXT:    v_mov_b32_e32 v2, s2
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    v_lshlrev_b16 v1, v2, v1
; GFX1030-NEXT:    global_store_short v0, v1, s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-TRUE16-LABEL: dpp_shl_i16_amount:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_clause 0x1
; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x2c
; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_lshlrev_b32 v1, 1, v0
; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1]
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
; GFX11-TRUE16-NEXT:    s_endpgm
;
; GFX11-FAKE16-LABEL: dpp_shl_i16_amount:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1]
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT:    v_lshlrev_b16_e64_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
; GFX11-FAKE16-NEXT:    s_endpgm
;
; GFX12-TRUE16-LABEL: dpp_shl_i16_amount:
; GFX12-TRUE16:       ; %bb.0:
; GFX12-TRUE16-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_lshlrev_b32 v1, 1, v0
; GFX12-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1]
; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-TRUE16-NEXT:    v_lshlrev_b16 v0.l, v2.l, v0.l
; GFX12-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
; GFX12-TRUE16-NEXT:    s_endpgm
;
; GFX12-FAKE16-LABEL: dpp_shl_i16_amount:
; GFX12-FAKE16:       ; %bb.0:
; GFX12-FAKE16-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1]
; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT:    v_lshlrev_b16_e64_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
; GFX12-FAKE16-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds i16, ptr addrspace(1) %arg, i32 %id
  %load = load i16, ptr addrspace(1) %gep
  %dpp = call i16 @llvm.amdgcn.update.dpp.i16(i16 %in, i16 %load, i32 1, i32 15, i32 15, i1 true)
  %shl = shl i16 %load, %dpp
  store i16 %shl, ptr addrspace(1) %gep
  ret void
}

; Same for v_lshrrev_b16.
define amdgpu_kernel void @dpp_lshr_i16_amount(ptr addrspace(1) %arg, i16 %in) {
; GFX8-LABEL: dpp_lshr_i16_amount:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
; GFX8-NEXT:    v_mov_b32_e32 v3, s2
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_mov_b32_dpp v3, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    v_lshrrev_b16_e32 v2, v3, v2
; GFX8-NEXT:    flat_store_short v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_lshr_i16_amount:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX9-NEXT:    v_mov_b32_e32 v2, s2
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    v_lshrrev_b16_e32 v1, v2, v1
; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_lshr_i16_amount:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX10-NEXT:    v_mov_b32_e32 v2, s2
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    v_lshrrev_b16 v1, v2, v1
; GFX10-NEXT:    global_store_short v0, v1, s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_lshr_i16_amount:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_clause 0x1
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX1030-NEXT:    v_mov_b32_e32 v2, s2
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    v_lshrrev_b16 v1, v2, v1
; GFX1030-NEXT:    global_store_short v0, v1, s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-TRUE16-LABEL: dpp_lshr_i16_amount:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_clause 0x1
; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x2c
; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_lshlrev_b32 v1, 1, v0
; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1]
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-TRUE16-NEXT:    v_lshrrev_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
; GFX11-TRUE16-NEXT:    s_endpgm
;
; GFX11-FAKE16-LABEL: dpp_lshr_i16_amount:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1]
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT:    v_lshrrev_b16_e64_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
; GFX11-FAKE16-NEXT:    s_endpgm
;
; GFX12-TRUE16-LABEL: dpp_lshr_i16_amount:
; GFX12-TRUE16:       ; %bb.0:
; GFX12-TRUE16-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_lshlrev_b32 v1, 1, v0
; GFX12-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1]
; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-TRUE16-NEXT:    v_lshrrev_b16 v0.l, v2.l, v0.l
; GFX12-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
; GFX12-TRUE16-NEXT:    s_endpgm
;
; GFX12-FAKE16-LABEL: dpp_lshr_i16_amount:
; GFX12-FAKE16:       ; %bb.0:
; GFX12-FAKE16-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1]
; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT:    v_lshrrev_b16_e64_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
; GFX12-FAKE16-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds i16, ptr addrspace(1) %arg, i32 %id
  %load = load i16, ptr addrspace(1) %gep
  %dpp = call i16 @llvm.amdgcn.update.dpp.i16(i16 %in, i16 %load, i32 1, i32 15, i32 15, i1 true)
  %shr = lshr i16 %load, %dpp
  store i16 %shr, ptr addrspace(1) %gep
  ret void
}

; Same for v_ashrrev_i16.
define amdgpu_kernel void @dpp_ashr_i16_amount(ptr addrspace(1) %arg, i16 %in) {
; GFX8-LABEL: dpp_ashr_i16_amount:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
; GFX8-NEXT:    v_mov_b32_e32 v3, s2
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_mov_b32_dpp v3, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    v_ashrrev_i16_e32 v2, v3, v2
; GFX8-NEXT:    flat_store_short v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_ashr_i16_amount:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX9-NEXT:    v_mov_b32_e32 v2, s2
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    v_ashrrev_i16_e32 v1, v2, v1
; GFX9-NEXT:    global_store_short v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_ashr_i16_amount:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX10-NEXT:    v_mov_b32_e32 v2, s2
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    v_ashrrev_i16 v1, v2, v1
; GFX10-NEXT:    global_store_short v0, v1, s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_ashr_i16_amount:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_clause 0x1
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_ushort v1, v0, s[0:1]
; GFX1030-NEXT:    v_mov_b32_e32 v2, s2
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    v_ashrrev_i16 v1, v2, v1
; GFX1030-NEXT:    global_store_short v0, v1, s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-TRUE16-LABEL: dpp_ashr_i16_amount:
; GFX11-TRUE16:       ; %bb.0:
; GFX11-TRUE16-NEXT:    s_clause 0x1
; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-TRUE16-NEXT:    s_load_b32 s2, s[4:5], 0x2c
; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_lshlrev_b32 v1, 1, v0
; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1]
; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-TRUE16-NEXT:    v_ashrrev_i16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
; GFX11-TRUE16-NEXT:    s_endpgm
;
; GFX11-FAKE16-LABEL: dpp_ashr_i16_amount:
; GFX11-FAKE16:       ; %bb.0:
; GFX11-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1]
; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT:    v_ashrrev_i16_e64_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
; GFX11-FAKE16-NEXT:    s_endpgm
;
; GFX12-TRUE16-LABEL: dpp_ashr_i16_amount:
; GFX12-TRUE16:       ; %bb.0:
; GFX12-TRUE16-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-TRUE16-NEXT:    s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_lshlrev_b32 v1, 1, v0
; GFX12-TRUE16-NEXT:    global_load_d16_b16 v0, v1, s[0:1]
; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT:    v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-TRUE16-NEXT:    v_ashrrev_i16 v0.l, v2.l, v0.l
; GFX12-TRUE16-NEXT:    global_store_b16 v1, v0, s[0:1]
; GFX12-TRUE16-NEXT:    s_endpgm
;
; GFX12-FAKE16-LABEL: dpp_ashr_i16_amount:
; GFX12-FAKE16:       ; %bb.0:
; GFX12-FAKE16-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT:    global_load_u16 v1, v0, s[0:1]
; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT:    v_ashrrev_i16_e64_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
; GFX12-FAKE16-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds i16, ptr addrspace(1) %arg, i32 %id
  %load = load i16, ptr addrspace(1) %gep
  %dpp = call i16 @llvm.amdgcn.update.dpp.i16(i16 %in, i16 %load, i32 1, i32 15, i32 15, i1 true)
  %shr = ashr i16 %load, %dpp
  store i16 %shr, ptr addrspace(1) %gep
  ret void
}

; Same for the 64 bit shift, v_lshlrev_b64, on the subtargets that have 64 bit
; DPP at all.
define amdgpu_kernel void @dpp_shl_i64_amount(ptr addrspace(1) %arg, i32 %in) {
; GFX8-LABEL: dpp_shl_i64_amount:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 3, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v2, s1
; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s0, v1
; GFX8-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
; GFX8-NEXT:    flat_load_dwordx2 v[3:4], v[1:2]
; GFX8-NEXT:    v_mov_b32_e32 v5, s2
; GFX8-NEXT:    s_nop 1
; GFX8-NEXT:    v_mov_b32_dpp v5, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    v_lshlrev_b64 v[3:4], v5, v[3:4]
; GFX8-NEXT:    flat_store_dwordx2 v[1:2], v[3:4]
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_shl_i64_amount:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 3, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_dwordx2 v[1:2], v3, s[0:1]
; GFX9-NEXT:    v_mov_b32_e32 v4, s2
; GFX9-NEXT:    s_nop 1
; GFX9-NEXT:    v_mov_b32_dpp v4, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    v_lshlrev_b64 v[0:1], v4, v[1:2]
; GFX9-NEXT:    global_store_dwordx2 v3, v[0:1], s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_shl_i64_amount:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 3, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_dwordx2 v[1:2], v3, s[0:1]
; GFX10-NEXT:    v_mov_b32_e32 v4, s2
; GFX10-NEXT:    v_mov_b32_dpp v4, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_lshlrev_b64 v[0:1], v4, v[1:2]
; GFX10-NEXT:    global_store_dwordx2 v3, v[0:1], s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_shl_i64_amount:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_clause 0x1
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX1030-NEXT:    v_lshlrev_b32_e32 v3, 3, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_dwordx2 v[1:2], v3, s[0:1]
; GFX1030-NEXT:    v_mov_b32_e32 v4, s2
; GFX1030-NEXT:    v_mov_b32_dpp v4, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_lshlrev_b64 v[0:1], v4, v[1:2]
; GFX1030-NEXT:    global_store_dwordx2 v3, v[0:1], s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-LABEL: dpp_shl_i64_amount:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_clause 0x1
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
; GFX11-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-NEXT:    v_dual_mov_b32 v4, s2 :: v_dual_lshlrev_b32 v3, 3, v2
; GFX11-NEXT:    global_load_b64 v[0:1], v3, s[0:1]
; GFX11-NEXT:    v_mov_b32_dpp v4, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_lshlrev_b64 v[0:1], v4, v[0:1]
; GFX11-NEXT:    global_store_b64 v3, v[0:1], s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: dpp_shl_i64_amount:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-NEXT:    v_dual_mov_b32 v4, s2 :: v_dual_lshlrev_b32 v3, 3, v2
; GFX12-NEXT:    global_load_b64 v[0:1], v3, s[0:1]
; GFX12-NEXT:    v_mov_b32_dpp v4, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_lshlrev_b64_e32 v[0:1], v4, v[0:1]
; GFX12-NEXT:    global_store_b64 v3, v[0:1], s[0:1]
; GFX12-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds i64, ptr addrspace(1) %arg, i32 %id
  %load = load i64, ptr addrspace(1) %gep
  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in, i32 %id, i32 1, i32 15, i32 15, i1 true)
  %ext = zext i32 %dpp to i64
  %shl = shl i64 %load, %ext
  store i64 %shl, ptr addrspace(1) %gep
  ret void
}

; The DPP value is the shifted operand rather than the shift amount, so it is
; src1 of v_lshlrev_b32. There is no non-REV shift to commute into, so this
; never folded to begin with.
define amdgpu_kernel void @dpp_shl_value(ptr addrspace(1) %arg, i32 %in) {
; GFX8-LABEL: dpp_shl_value:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_dword v2, v[0:1]
; GFX8-NEXT:    v_mov_b32_e32 v3, s2
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_mov_b32_dpp v3, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    v_lshlrev_b32_e32 v2, v2, v3
; GFX8-NEXT:    flat_store_dword v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_shl_value:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX9-NEXT:    v_mov_b32_e32 v2, s2
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    v_lshlrev_b32_e32 v1, v1, v2
; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_shl_value:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_clause 0x1
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX10-NEXT:    v_mov_b32_e32 v2, s2
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    v_lshlrev_b32_e32 v1, v1, v2
; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_shl_value:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_clause 0x1
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    s_load_dword s2, s[4:5], 0x2c
; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX1030-NEXT:    v_mov_b32_e32 v2, s2
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    v_lshlrev_b32_e32 v1, v1, v2
; GFX1030-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-LABEL: dpp_shl_value:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_clause 0x1
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX11-NEXT:    v_mov_b32_e32 v2, s2
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_lshlrev_b32_e32 v1, v1, v2
; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: dpp_shl_value:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT:    v_mov_b32_e32 v2, s2
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_mov_b32_dpp v2, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_lshlrev_b32_e32 v1, v1, v2
; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds i32, ptr addrspace(1) %arg, i32 %id
  %load = load i32, ptr addrspace(1) %gep
  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in, i32 %load, i32 1, i32 15, i32 15, i1 true)
  %shl = shl i32 %dpp, %load
  store i32 %shl, ptr addrspace(1) %gep
  ret void
}

; Commuting into an opcode that is its own reverse keeps folding.
define amdgpu_kernel void @dpp_add_u32(ptr addrspace(1) %arg, i32 %in) {
; GFX8-LABEL: dpp_add_u32:
; GFX8:       ; %bb.0:
; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
; GFX8-NEXT:    v_mov_b32_e32 v1, s1
; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT:    flat_load_dword v2, v[0:1]
; GFX8-NEXT:    s_waitcnt vmcnt(0)
; GFX8-NEXT:    s_nop 0
; GFX8-NEXT:    v_add_u32_dpp v2, vcc, v2, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX8-NEXT:    flat_store_dword v[0:1], v2
; GFX8-NEXT:    s_endpgm
;
; GFX9-LABEL: dpp_add_u32:
; GFX9:       ; %bb.0:
; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
; GFX9-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX9-NEXT:    s_waitcnt vmcnt(0)
; GFX9-NEXT:    s_nop 0
; GFX9-NEXT:    v_add_u32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX9-NEXT:    s_endpgm
;
; GFX10-LABEL: dpp_add_u32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_add_nc_u32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX10-NEXT:    s_endpgm
;
; GFX1030-LABEL: dpp_add_u32:
; GFX1030:       ; %bb.0:
; GFX1030-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX1030-NEXT:    s_waitcnt lgkmcnt(0)
; GFX1030-NEXT:    global_load_dword v1, v0, s[0:1]
; GFX1030-NEXT:    s_waitcnt vmcnt(0)
; GFX1030-NEXT:    v_add_nc_u32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1030-NEXT:    global_store_dword v0, v1, s[0:1]
; GFX1030-NEXT:    s_endpgm
;
; GFX11-LABEL: dpp_add_u32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    v_add_nc_u32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT:    s_endpgm
;
; GFX12-LABEL: dpp_add_u32:
; GFX12:       ; %bb.0:
; GFX12-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; GFX12-NEXT:    s_wait_kmcnt 0x0
; GFX12-NEXT:    global_load_b32 v1, v0, s[0:1]
; GFX12-NEXT:    s_wait_loadcnt 0x0
; GFX12-NEXT:    v_add_nc_u32_dpp v1, v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT:    s_endpgm
  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
  %gep = getelementptr inbounds i32, ptr addrspace(1) %arg, i32 %id
  %load = load i32, ptr addrspace(1) %gep
  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in, i32 %load, i32 1, i32 15, i32 15, i1 true)
  %add = add i32 %load, %dpp
  store i32 %add, ptr addrspace(1) %gep
  ret void
}

declare i32 @llvm.amdgcn.workitem.id.x()