llvm/test/CodeGen/AMDGPU/idot4-test.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu9.06 < %s | FileCheck --check-prefixes=GFX9-DL %s
; RUN: llc -mtriple=amdgpu10.11 < %s | FileCheck --check-prefixes=GFX10-DL %s
; RUN: llc -mtriple=amdgpu9.50 < %s | FileCheck --check-prefixes=GFX950 %s
; RUN: llc -mtriple=amdgpu9.00 < %s | FileCheck --check-prefixes=GFX9-NODL %s
; RUN: opt -mtriple=amdgpu9.06 -passes=amdgpu-codegenprepare -S < %s | FileCheck --check-prefixes=OPT %s

; Test dot4 patterns with saturating add (clamp) and without

;------------------------------------------------------------------------------
; DOT4 TESTS WITH <4 x i8> ARGS AND VECTOR REDUCE
;------------------------------------------------------------------------------

define i32 @test_udot4_sat(<4 x i8> %a, <4 x i8> %b, i32 %c) {
; GFX9-DL-LABEL: test_udot4_sat:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    s_mov_b32 s4, 0xc0c0004
; GFX9-DL-NEXT:    v_perm_b32 v0, v0, v1, s4
; GFX9-DL-NEXT:    v_perm_b32 v1, v2, v3, s4
; GFX9-DL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX9-DL-NEXT:    v_perm_b32 v1, v4, v5, s4
; GFX9-DL-NEXT:    v_perm_b32 v2, v6, v7, s4
; GFX9-DL-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
; GFX9-DL-NEXT:    v_dot4_u32_u8 v0, v1, v0, v8 clamp
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: test_udot4_sat:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_perm_b32 v0, v0, v1, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX10-DL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX10-DL-NEXT:    v_lshl_or_b32 v1, v3, 16, v2
; GFX10-DL-NEXT:    v_dot4_u32_u8 v0, v1, v0, v8 clamp
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: test_udot4_sat:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    s_mov_b32 s0, 0xc0c0004
; GFX950-NEXT:    v_perm_b32 v0, v0, v1, s0
; GFX950-NEXT:    v_perm_b32 v1, v2, v3, s0
; GFX950-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX950-NEXT:    v_perm_b32 v1, v4, v5, s0
; GFX950-NEXT:    v_perm_b32 v2, v6, v7, s0
; GFX950-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
; GFX950-NEXT:    v_dot4_u32_u8 v0, v1, v0, v8 clamp
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: test_udot4_sat:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v2, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_add_u32_e32 v1, v1, v3
; GFX9-NODL-NEXT:    v_add3_u32 v0, v0, v2, v1
; GFX9-NODL-NEXT:    v_add_u32_e64 v0, v0, v8 clamp
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
entry:
  %conv.i = zext <4 x i8> %a to <4 x i32>
  %conv8.i = zext <4 x i8> %b to <4 x i32>
  %mul.i = mul <4 x i32> %conv8.i, %conv.i
  %rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i)
  %cond.i.i = tail call i32 @llvm.uadd.sat.i32(i32 %rdx.add.i, i32 %c)
  ret i32 %cond.i.i
}

define i32 @test_udot4_unsat(<4 x i8> %a, <4 x i8> %b, i32 %c) {
; GFX9-DL-LABEL: test_udot4_unsat:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    s_mov_b32 s4, 0xc0c0004
; GFX9-DL-NEXT:    v_perm_b32 v0, v0, v1, s4
; GFX9-DL-NEXT:    v_perm_b32 v1, v2, v3, s4
; GFX9-DL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX9-DL-NEXT:    v_perm_b32 v1, v4, v5, s4
; GFX9-DL-NEXT:    v_perm_b32 v2, v6, v7, s4
; GFX9-DL-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
; GFX9-DL-NEXT:    v_dot4_u32_u8 v0, v1, v0, 0
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v0, v8
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: test_udot4_unsat:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_perm_b32 v0, v0, v1, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX10-DL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX10-DL-NEXT:    v_lshl_or_b32 v1, v3, 16, v2
; GFX10-DL-NEXT:    v_dot4_u32_u8 v0, v1, v0, 0
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v0, v8
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: test_udot4_unsat:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    s_mov_b32 s0, 0xc0c0004
; GFX950-NEXT:    v_perm_b32 v0, v0, v1, s0
; GFX950-NEXT:    v_perm_b32 v1, v2, v3, s0
; GFX950-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX950-NEXT:    v_perm_b32 v1, v4, v5, s0
; GFX950-NEXT:    v_perm_b32 v2, v6, v7, s0
; GFX950-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
; GFX950-NEXT:    v_dot4_u32_u8 v0, v1, v0, 0
; GFX950-NEXT:    s_nop 2
; GFX950-NEXT:    v_add_u32_e32 v0, v0, v8
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: test_udot4_unsat:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v2, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_add_u32_e32 v1, v1, v3
; GFX9-NODL-NEXT:    v_add_u32_e32 v0, v0, v2
; GFX9-NODL-NEXT:    v_add3_u32 v0, v0, v1, v8
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
entry:
  %conv.i = zext <4 x i8> %a to <4 x i32>
  %conv8.i = zext <4 x i8> %b to <4 x i32>
  %mul.i = mul <4 x i32> %conv8.i, %conv.i
  %rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i)
  %add.i = add i32 %rdx.add.i, %c
  ret i32 %add.i
}

define i32 @test_sdot4_sat(<4 x i8> %a, <4 x i8> %b, i32 %c) {
; GFX9-DL-LABEL: test_sdot4_sat:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    s_mov_b32 s4, 0xc0c0004
; GFX9-DL-NEXT:    v_perm_b32 v0, v0, v1, s4
; GFX9-DL-NEXT:    v_perm_b32 v1, v2, v3, s4
; GFX9-DL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX9-DL-NEXT:    v_perm_b32 v1, v4, v5, s4
; GFX9-DL-NEXT:    v_perm_b32 v2, v6, v7, s4
; GFX9-DL-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
; GFX9-DL-NEXT:    v_dot4_i32_i8 v0, v1, v0, v8 clamp
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: test_sdot4_sat:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_perm_b32 v0, v0, v1, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX10-DL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX10-DL-NEXT:    v_lshl_or_b32 v1, v3, 16, v2
; GFX10-DL-NEXT:    v_dot4_i32_i8 v0, v1, v0, v8 clamp
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: test_sdot4_sat:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    s_mov_b32 s0, 0xc0c0004
; GFX950-NEXT:    v_perm_b32 v0, v0, v1, s0
; GFX950-NEXT:    v_perm_b32 v1, v2, v3, s0
; GFX950-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX950-NEXT:    v_perm_b32 v1, v4, v5, s0
; GFX950-NEXT:    v_perm_b32 v2, v6, v7, s0
; GFX950-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
; GFX950-NEXT:    v_dot4_i32_i8 v0, v1, v0, v8 clamp
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: test_sdot4_sat:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v1, sext(v5), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v3, sext(v7), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v4), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v2, sext(v6), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_add_u32_e32 v1, v1, v3
; GFX9-NODL-NEXT:    v_add3_u32 v0, v0, v2, v1
; GFX9-NODL-NEXT:    v_add_i32 v0, v0, v8 clamp
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
entry:
  %conv.i = sext <4 x i8> %a to <4 x i32>
  %conv8.i = sext <4 x i8> %b to <4 x i32>
  %mul.i = mul <4 x i32> %conv8.i, %conv.i
  %rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i)
  %cond1.i.i = tail call i32 @llvm.sadd.sat.i32(i32 %rdx.add.i, i32 %c)
  ret i32 %cond1.i.i
}

define i32 @test_sdot4_unsat(<4 x i8> %a, <4 x i8> %b, i32 %c) {
; GFX9-DL-LABEL: test_sdot4_unsat:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    s_mov_b32 s4, 0xc0c0004
; GFX9-DL-NEXT:    v_perm_b32 v0, v0, v1, s4
; GFX9-DL-NEXT:    v_perm_b32 v1, v2, v3, s4
; GFX9-DL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX9-DL-NEXT:    v_perm_b32 v1, v4, v5, s4
; GFX9-DL-NEXT:    v_perm_b32 v2, v6, v7, s4
; GFX9-DL-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
; GFX9-DL-NEXT:    v_dot4_i32_i8 v0, v1, v0, 0
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v0, v8
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: test_sdot4_unsat:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_perm_b32 v0, v0, v1, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX10-DL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX10-DL-NEXT:    v_lshl_or_b32 v1, v3, 16, v2
; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT:    v_dot4c_i32_i8 v2, v1, v0
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v2, v8
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: test_sdot4_unsat:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    s_mov_b32 s0, 0xc0c0004
; GFX950-NEXT:    v_perm_b32 v0, v0, v1, s0
; GFX950-NEXT:    v_perm_b32 v1, v2, v3, s0
; GFX950-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX950-NEXT:    v_perm_b32 v1, v4, v5, s0
; GFX950-NEXT:    v_perm_b32 v2, v6, v7, s0
; GFX950-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
; GFX950-NEXT:    v_mov_b32_e32 v2, 0
; GFX950-NEXT:    v_dot4c_i32_i8_e32 v2, v1, v0
; GFX950-NEXT:    s_nop 2
; GFX950-NEXT:    v_add_u32_e32 v0, v2, v8
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: test_sdot4_unsat:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v4), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v1, sext(v5), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v2, sext(v6), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v3, sext(v7), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_add_u32_e32 v1, v1, v3
; GFX9-NODL-NEXT:    v_add_u32_e32 v0, v0, v2
; GFX9-NODL-NEXT:    v_add3_u32 v0, v0, v1, v8
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
entry:
  %conv.i = sext <4 x i8> %a to <4 x i32>
  %conv8.i = sext <4 x i8> %b to <4 x i32>
  %mul.i = mul <4 x i32> %conv8.i, %conv.i
  %rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i)
  %add.i = add i32 %rdx.add.i, %c
  ret i32 %add.i
}

;------------------------------------------------------------------------------
; DOT4 TESTS WITH BITCAST FROM I32
;------------------------------------------------------------------------------

; Unsigned dot4 without saturation using bitcast from i32
; OPT-LABEL: @udot4_unsat_bitcast(
; OPT:         [[A:%.*]] = bitcast i32 %a_packed to <4 x i8>
; OPT:         [[B:%.*]] = bitcast i32 %b_packed to <4 x i8>
; OPT:         [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32
; OPT:         [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32
; OPT:         [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A32]], i32 [[B32]], i32 0, i1 false)
; OPT:         [[RES:%.*]] = add i32 [[DOT]], %c
; OPT:         ret i32 [[RES]]
;
define i32 @udot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-DL-LABEL: udot4_unsat_bitcast:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    v_dot4_u32_u8 v0, v0, v1, 0
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v0, v2
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: udot4_unsat_bitcast:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_dot4_u32_u8 v0, v0, v1, 0
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v0, v2
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: udot4_unsat_bitcast:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    v_dot4_u32_u8 v0, v0, v1, 0
; GFX950-NEXT:    s_nop 2
; GFX950-NEXT:    v_add_u32_e32 v0, v0, v2
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: udot4_unsat_bitcast:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v3, 8, v0
; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v4, 8, v1
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, v3, v4, v0
; GFX9-NODL-NEXT:    v_add_u32_e32 v1, v5, v6
; GFX9-NODL-NEXT:    v_add3_u32 v0, v1, v0, v2
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
entry:
  %a_vec = bitcast i32 %a_packed to <4 x i8>
  %b_vec = bitcast i32 %b_packed to <4 x i8>
  %a_ext = zext <4 x i8> %a_vec to <4 x i32>
  %b_ext = zext <4 x i8> %b_vec to <4 x i32>
  %mul = mul <4 x i32> %a_ext, %b_ext
  %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
  %result = add i32 %sum, %c
  ret i32 %result
}

; Signed dot4 without saturation using bitcast from i32
; OPT-LABEL: @sdot4_unsat_bitcast(
; OPT:         [[A:%.*]] = bitcast i32 %a_packed to <4 x i8>
; OPT:         [[B:%.*]] = bitcast i32 %b_packed to <4 x i8>
; OPT:         [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32
; OPT:         [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32
; OPT:         [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A32]], i32 [[B32]], i32 0, i1 false)
; OPT:         [[RES:%.*]] = add i32 [[DOT]], %c
; OPT:         ret i32 [[RES]]
;
define i32 @sdot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-DL-LABEL: sdot4_unsat_bitcast:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    v_dot4_i32_i8 v0, v0, v1, 0
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v0, v2
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: sdot4_unsat_bitcast:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_mov_b32_e32 v3, 0
; GFX10-DL-NEXT:    v_dot4c_i32_i8 v3, v0, v1
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v3, v2
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: sdot4_unsat_bitcast:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    v_mov_b32_e32 v3, 0
; GFX950-NEXT:    v_dot4c_i32_i8_e32 v3, v0, v1
; GFX950-NEXT:    s_nop 2
; GFX950-NEXT:    v_add_u32_e32 v0, v3, v2
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: sdot4_unsat_bitcast:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v3, 8, v1
; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v4, 8, v0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT:    v_add_u32_e32 v0, v3, v0
; GFX9-NODL-NEXT:    v_add_u32_e32 v1, v5, v4
; GFX9-NODL-NEXT:    v_add3_u32 v0, v1, v0, v2
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
entry:
  %a_vec = bitcast i32 %a_packed to <4 x i8>
  %b_vec = bitcast i32 %b_packed to <4 x i8>
  %a_ext = sext <4 x i8> %a_vec to <4 x i32>
  %b_ext = sext <4 x i8> %b_vec to <4 x i32>
  %mul = mul <4 x i32> %a_ext, %b_ext
  %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
  %result = add i32 %sum, %c
  ret i32 %result
}

; Unsigned dot4 with saturation using bitcast from i32
; OPT-LABEL: @udot4_sat_bitcast(
; OPT:         [[A:%.*]] = bitcast i32 %a_packed to <4 x i8>
; OPT:         [[B:%.*]] = bitcast i32 %b_packed to <4 x i8>
; OPT:         [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32
; OPT:         [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32
; OPT:         [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A32]], i32 [[B32]], i32 %c, i1 true)
; OPT:         ret i32 [[DOT]]
;
define i32 @udot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-DL-LABEL: udot4_sat_bitcast:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    v_dot4_u32_u8 v0, v0, v1, v2 clamp
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: udot4_sat_bitcast:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_dot4_u32_u8 v0, v0, v1, v2 clamp
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: udot4_sat_bitcast:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    v_dot4_u32_u8 v0, v0, v1, v2 clamp
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: udot4_sat_bitcast:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v3, 8, v0
; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v4, 8, v1
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, v3, v4, v0
; GFX9-NODL-NEXT:    v_add3_u32 v0, v5, v6, v0
; GFX9-NODL-NEXT:    v_add_u32_e64 v0, v0, v2 clamp
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
entry:
  %a_vec = bitcast i32 %a_packed to <4 x i8>
  %b_vec = bitcast i32 %b_packed to <4 x i8>
  %a_ext = zext <4 x i8> %a_vec to <4 x i32>
  %b_ext = zext <4 x i8> %b_vec to <4 x i32>
  %mul = mul <4 x i32> %a_ext, %b_ext
  %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
  %result = call i32 @llvm.uadd.sat.i32(i32 %sum, i32 %c)
  ret i32 %result
}

; Signed dot4 with saturation using bitcast from i32
; OPT-LABEL: @sdot4_sat_bitcast(
; OPT:         [[A:%.*]] = bitcast i32 %a_packed to <4 x i8>
; OPT:         [[B:%.*]] = bitcast i32 %b_packed to <4 x i8>
; OPT:         [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32
; OPT:         [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32
; OPT:         [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A32]], i32 [[B32]], i32 %c, i1 true)
; OPT:         ret i32 [[DOT]]
;
define i32 @sdot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-DL-LABEL: sdot4_sat_bitcast:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    v_dot4_i32_i8 v0, v0, v1, v2 clamp
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: sdot4_sat_bitcast:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_dot4_i32_i8 v0, v0, v1, v2 clamp
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: sdot4_sat_bitcast:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    v_dot4_i32_i8 v0, v0, v1, v2 clamp
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: sdot4_sat_bitcast:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v3, 8, v1
; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v4, 8, v0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT:    v_add_u32_e32 v0, v3, v0
; GFX9-NODL-NEXT:    v_add3_u32 v0, v5, v4, v0
; GFX9-NODL-NEXT:    v_add_i32 v0, v0, v2 clamp
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
entry:
  %a_vec = bitcast i32 %a_packed to <4 x i8>
  %b_vec = bitcast i32 %b_packed to <4 x i8>
  %a_ext = sext <4 x i8> %a_vec to <4 x i32>
  %b_ext = sext <4 x i8> %b_vec to <4 x i32>
  %mul = mul <4 x i32> %a_ext, %b_ext
  %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
  %result = call i32 @llvm.sadd.sat.i32(i32 %sum, i32 %c)
  ret i32 %result
}

;------------------------------------------------------------------------------
; TESTS WITH <4 x i8> FUNCTION ARGUMENTS
;------------------------------------------------------------------------------

; Test with raw <4 x i8> arguments (ABI passes as separate registers)
define i32 @udot4_v4i8_arg(<4 x i8> %a, <4 x i8> %b, i32 %c) {
; GFX9-DL-LABEL: udot4_v4i8_arg:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    s_mov_b32 s4, 0xc0c0004
; GFX9-DL-NEXT:    v_perm_b32 v4, v4, v5, s4
; GFX9-DL-NEXT:    v_perm_b32 v5, v6, v7, s4
; GFX9-DL-NEXT:    v_perm_b32 v0, v0, v1, s4
; GFX9-DL-NEXT:    v_perm_b32 v1, v2, v3, s4
; GFX9-DL-NEXT:    v_lshl_or_b32 v4, v5, 16, v4
; GFX9-DL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX9-DL-NEXT:    v_dot4_u32_u8 v0, v0, v4, 0
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v0, v8
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: udot4_v4i8_arg:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_perm_b32 v4, v4, v5, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v5, v6, v7, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v0, v0, v1, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX10-DL-NEXT:    v_lshl_or_b32 v2, v5, 16, v4
; GFX10-DL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX10-DL-NEXT:    v_dot4_u32_u8 v0, v0, v2, 0
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v0, v8
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: udot4_v4i8_arg:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    s_mov_b32 s0, 0xc0c0004
; GFX950-NEXT:    v_perm_b32 v4, v4, v5, s0
; GFX950-NEXT:    v_perm_b32 v5, v6, v7, s0
; GFX950-NEXT:    v_perm_b32 v0, v0, v1, s0
; GFX950-NEXT:    v_perm_b32 v1, v2, v3, s0
; GFX950-NEXT:    v_lshl_or_b32 v4, v5, 16, v4
; GFX950-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX950-NEXT:    v_dot4_u32_u8 v0, v0, v4, 0
; GFX950-NEXT:    s_nop 2
; GFX950-NEXT:    v_add_u32_e32 v0, v0, v8
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: udot4_v4i8_arg:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_add_u32_e32 v1, v1, v3
; GFX9-NODL-NEXT:    v_add_u32_e32 v0, v0, v2
; GFX9-NODL-NEXT:    v_add3_u32 v0, v0, v1, v8
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
entry:
  %a_ext = zext <4 x i8> %a to <4 x i32>
  %b_ext = zext <4 x i8> %b to <4 x i32>
  %mul = mul <4 x i32> %a_ext, %b_ext
  %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
  %result = add i32 %sum, %c
  ret i32 %result
}

define i32 @sdot4_v4i8_arg(<4 x i8> %a, <4 x i8> %b, i32 %c) {
; GFX9-DL-LABEL: sdot4_v4i8_arg:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    s_mov_b32 s4, 0xc0c0004
; GFX9-DL-NEXT:    v_perm_b32 v4, v4, v5, s4
; GFX9-DL-NEXT:    v_perm_b32 v5, v6, v7, s4
; GFX9-DL-NEXT:    v_perm_b32 v0, v0, v1, s4
; GFX9-DL-NEXT:    v_perm_b32 v1, v2, v3, s4
; GFX9-DL-NEXT:    v_lshl_or_b32 v4, v5, 16, v4
; GFX9-DL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX9-DL-NEXT:    v_dot4_i32_i8 v0, v0, v4, 0
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v0, v8
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: sdot4_v4i8_arg:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_perm_b32 v4, v4, v5, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v5, v6, v7, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v0, v0, v1, 0xc0c0004
; GFX10-DL-NEXT:    v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX10-DL-NEXT:    v_lshl_or_b32 v2, v5, 16, v4
; GFX10-DL-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT:    v_dot4c_i32_i8 v1, v0, v2
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v1, v8
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: sdot4_v4i8_arg:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    s_mov_b32 s0, 0xc0c0004
; GFX950-NEXT:    v_perm_b32 v4, v4, v5, s0
; GFX950-NEXT:    v_perm_b32 v5, v6, v7, s0
; GFX950-NEXT:    v_perm_b32 v0, v0, v1, s0
; GFX950-NEXT:    v_perm_b32 v1, v2, v3, s0
; GFX950-NEXT:    v_lshl_or_b32 v4, v5, 16, v4
; GFX950-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
; GFX950-NEXT:    v_mov_b32_e32 v1, 0
; GFX950-NEXT:    v_dot4c_i32_i8_e32 v1, v0, v4
; GFX950-NEXT:    s_nop 2
; GFX950-NEXT:    v_add_u32_e32 v0, v1, v8
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: sdot4_v4i8_arg:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_add_u32_e32 v1, v1, v3
; GFX9-NODL-NEXT:    v_add_u32_e32 v0, v0, v2
; GFX9-NODL-NEXT:    v_add3_u32 v0, v0, v1, v8
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
entry:
  %a_ext = sext <4 x i8> %a to <4 x i32>
  %b_ext = sext <4 x i8> %b to <4 x i32>
  %mul = mul <4 x i32> %a_ext, %b_ext
  %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
  %result = add i32 %sum, %c
  ret i32 %result
}

;------------------------------------------------------------------------------
; TESTS WITH I8 PROMOTION
;------------------------------------------------------------------------------

; Test unsigned dot4 with i8 values zext'd individually
define i32 @udot4_i8_zext(i8 %a0, i8 %a1, i8 %a2, i8 %a3,
; GFX9-DL-LABEL: udot4_i8_zext:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    s_mov_b32 s4, 0xc0c0400
; GFX9-DL-NEXT:    s_mov_b32 s5, 0x4000c0c
; GFX9-DL-NEXT:    v_perm_b32 v4, v4, v5, s4
; GFX9-DL-NEXT:    v_perm_b32 v5, v7, v6, s5
; GFX9-DL-NEXT:    v_perm_b32 v0, v0, v1, s4
; GFX9-DL-NEXT:    v_perm_b32 v1, v3, v2, s5
; GFX9-DL-NEXT:    v_or_b32_e32 v4, v5, v4
; GFX9-DL-NEXT:    v_or_b32_e32 v0, v1, v0
; GFX9-DL-NEXT:    v_dot4_u32_u8 v0, v0, v4, 0
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v0, v8
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: udot4_i8_zext:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_perm_b32 v4, v4, v5, 0xc0c0400
; GFX10-DL-NEXT:    v_perm_b32 v5, v7, v6, 0x4000c0c
; GFX10-DL-NEXT:    v_perm_b32 v0, v0, v1, 0xc0c0400
; GFX10-DL-NEXT:    v_perm_b32 v1, v3, v2, 0x4000c0c
; GFX10-DL-NEXT:    v_or_b32_e32 v2, v5, v4
; GFX10-DL-NEXT:    v_or_b32_e32 v0, v1, v0
; GFX10-DL-NEXT:    v_dot4_u32_u8 v0, v0, v2, 0
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v0, v8
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: udot4_i8_zext:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    s_mov_b32 s0, 0xc0c0400
; GFX950-NEXT:    s_mov_b32 s1, 0x4000c0c
; GFX950-NEXT:    v_perm_b32 v4, v4, v5, s0
; GFX950-NEXT:    v_perm_b32 v5, v7, v6, s1
; GFX950-NEXT:    v_perm_b32 v0, v0, v1, s0
; GFX950-NEXT:    v_perm_b32 v1, v3, v2, s1
; GFX950-NEXT:    v_or_b32_e32 v4, v5, v4
; GFX950-NEXT:    v_or_b32_e32 v0, v1, v0
; GFX950-NEXT:    v_dot4_u32_u8 v0, v0, v4, 0
; GFX950-NEXT:    s_nop 2
; GFX950-NEXT:    v_add_u32_e32 v0, v0, v8
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: udot4_i8_zext:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_add3_u32 v0, v0, v1, v2
; GFX9-NODL-NEXT:    v_add3_u32 v0, v0, v3, v8
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
                          i8 %b0, i8 %b1, i8 %b2, i8 %b3, i32 %c) {
entry:
  %a0_32 = zext i8 %a0 to i32
  %a1_32 = zext i8 %a1 to i32
  %a2_32 = zext i8 %a2 to i32
  %a3_32 = zext i8 %a3 to i32
  %b0_32 = zext i8 %b0 to i32
  %b1_32 = zext i8 %b1 to i32
  %b2_32 = zext i8 %b2 to i32
  %b3_32 = zext i8 %b3 to i32
  %m0 = mul i32 %a0_32, %b0_32
  %m1 = mul i32 %a1_32, %b1_32
  %m2 = mul i32 %a2_32, %b2_32
  %m3 = mul i32 %a3_32, %b3_32
  %sum01 = add i32 %m0, %m1
  %sum012 = add i32 %sum01, %m2
  %sum0123 = add i32 %sum012, %m3
  %result = add i32 %sum0123, %c
  ret i32 %result
}

; Test signed dot4 with i8 values sext'd individually
define i32 @sdot4_i8_sext(i8 %a0, i8 %a1, i8 %a2, i8 %a3,
; GFX9-DL-LABEL: sdot4_i8_sext:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    s_mov_b32 s4, 0xc0c0400
; GFX9-DL-NEXT:    s_mov_b32 s5, 0x4000c0c
; GFX9-DL-NEXT:    v_perm_b32 v4, v4, v5, s4
; GFX9-DL-NEXT:    v_perm_b32 v5, v7, v6, s5
; GFX9-DL-NEXT:    v_perm_b32 v0, v0, v1, s4
; GFX9-DL-NEXT:    v_perm_b32 v1, v3, v2, s5
; GFX9-DL-NEXT:    v_or_b32_e32 v4, v5, v4
; GFX9-DL-NEXT:    v_or_b32_e32 v0, v1, v0
; GFX9-DL-NEXT:    v_dot4_i32_i8 v0, v0, v4, 0
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v0, v8
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: sdot4_i8_sext:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_perm_b32 v4, v4, v5, 0xc0c0400
; GFX10-DL-NEXT:    v_perm_b32 v5, v7, v6, 0x4000c0c
; GFX10-DL-NEXT:    v_perm_b32 v0, v0, v1, 0xc0c0400
; GFX10-DL-NEXT:    v_perm_b32 v1, v3, v2, 0x4000c0c
; GFX10-DL-NEXT:    v_or_b32_e32 v2, v5, v4
; GFX10-DL-NEXT:    v_or_b32_e32 v0, v1, v0
; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
; GFX10-DL-NEXT:    v_dot4c_i32_i8 v1, v0, v2
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v1, v8
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: sdot4_i8_sext:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    s_mov_b32 s0, 0xc0c0400
; GFX950-NEXT:    s_mov_b32 s1, 0x4000c0c
; GFX950-NEXT:    v_perm_b32 v4, v4, v5, s0
; GFX950-NEXT:    v_perm_b32 v5, v7, v6, s1
; GFX950-NEXT:    v_perm_b32 v0, v0, v1, s0
; GFX950-NEXT:    v_perm_b32 v1, v3, v2, s1
; GFX950-NEXT:    v_or_b32_e32 v4, v5, v4
; GFX950-NEXT:    v_or_b32_e32 v0, v1, v0
; GFX950-NEXT:    v_mov_b32_e32 v1, 0
; GFX950-NEXT:    v_dot4c_i32_i8_e32 v1, v0, v4
; GFX950-NEXT:    s_nop 2
; GFX950-NEXT:    v_add_u32_e32 v0, v1, v8
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: sdot4_i8_sext:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_add3_u32 v0, v0, v1, v2
; GFX9-NODL-NEXT:    v_add3_u32 v0, v0, v3, v8
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
                          i8 %b0, i8 %b1, i8 %b2, i8 %b3, i32 %c) {
entry:
  %a0_32 = sext i8 %a0 to i32
  %a1_32 = sext i8 %a1 to i32
  %a2_32 = sext i8 %a2 to i32
  %a3_32 = sext i8 %a3 to i32
  %b0_32 = sext i8 %b0 to i32
  %b1_32 = sext i8 %b1 to i32
  %b2_32 = sext i8 %b2 to i32
  %b3_32 = sext i8 %b3 to i32
  %m0 = mul i32 %a0_32, %b0_32
  %m1 = mul i32 %a1_32, %b1_32
  %m2 = mul i32 %a2_32, %b2_32
  %m3 = mul i32 %a3_32, %b3_32
  %sum01 = add i32 %m0, %m1
  %sum012 = add i32 %sum01, %m2
  %sum0123 = add i32 %sum012, %m3
  %result = add i32 %sum0123, %c
  ret i32 %result
}

;------------------------------------------------------------------------------
; NEGATIVE TESTS
;------------------------------------------------------------------------------

; Negative test: mismatched signedness (zext with sadd.sat)
define i32 @dot4_mismatched_sign_zext_sadd(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-DL-LABEL: dot4_mismatched_sign_zext_sadd:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    v_dot4_u32_u8 v0, v0, v1, 0
; GFX9-DL-NEXT:    v_add_i32 v0, v0, v2 clamp
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: dot4_mismatched_sign_zext_sadd:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_dot4_u32_u8 v0, v0, v1, 0
; GFX10-DL-NEXT:    v_add_nc_i32 v0, v0, v2 clamp
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: dot4_mismatched_sign_zext_sadd:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    v_dot4_u32_u8 v0, v0, v1, 0
; GFX950-NEXT:    s_nop 2
; GFX950-NEXT:    v_add_i32 v0, v0, v2 clamp
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: dot4_mismatched_sign_zext_sadd:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v3, 8, v0
; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v4, 8, v1
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT:    v_mad_u32_u24 v0, v3, v4, v0
; GFX9-NODL-NEXT:    v_add3_u32 v0, v5, v6, v0
; GFX9-NODL-NEXT:    v_add_i32 v0, v0, v2 clamp
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
entry:
  %a_vec = bitcast i32 %a_packed to <4 x i8>
  %b_vec = bitcast i32 %b_packed to <4 x i8>
  %a_ext = zext <4 x i8> %a_vec to <4 x i32>
  %b_ext = zext <4 x i8> %b_vec to <4 x i32>
  %mul = mul <4 x i32> %a_ext, %b_ext
  %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
  %result = call i32 @llvm.sadd.sat.i32(i32 %sum, i32 %c)
  ret i32 %result
}

; Negative test: mismatched signedness (sext with uadd.sat)
define i32 @dot4_mismatched_sign_sext_uadd(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-DL-LABEL: dot4_mismatched_sign_sext_uadd:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    v_dot4_i32_i8 v0, v0, v1, 0
; GFX9-DL-NEXT:    v_add_u32_e64 v0, v0, v2 clamp
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: dot4_mismatched_sign_sext_uadd:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_mov_b32_e32 v3, 0
; GFX10-DL-NEXT:    v_dot4c_i32_i8 v3, v0, v1
; GFX10-DL-NEXT:    v_add_nc_u32_e64 v0, v3, v2 clamp
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: dot4_mismatched_sign_sext_uadd:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    v_mov_b32_e32 v3, 0
; GFX950-NEXT:    v_dot4c_i32_i8_e32 v3, v0, v1
; GFX950-NEXT:    s_nop 2
; GFX950-NEXT:    v_add_u32_e64 v0, v3, v2 clamp
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: dot4_mismatched_sign_sext_uadd:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v3, 8, v1
; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v4, 8, v0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT:    v_add_u32_e32 v0, v3, v0
; GFX9-NODL-NEXT:    v_add3_u32 v0, v5, v4, v0
; GFX9-NODL-NEXT:    v_add_u32_e64 v0, v0, v2 clamp
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
entry:
  %a_vec = bitcast i32 %a_packed to <4 x i8>
  %b_vec = bitcast i32 %b_packed to <4 x i8>
  %a_ext = sext <4 x i8> %a_vec to <4 x i32>
  %b_ext = sext <4 x i8> %b_vec to <4 x i32>
  %mul = mul <4 x i32> %a_ext, %b_ext
  %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
  %result = call i32 @llvm.uadd.sat.i32(i32 %sum, i32 %c)
  ret i32 %result
}

; Negative test: mixed zext/sext operands
define i32 @dot4_mixed_ext(i32 %a_packed, i32 %b_packed, i32 %c) {
; GFX9-DL-LABEL: dot4_mixed_ext:
; GFX9-DL:       ; %bb.0: ; %entry
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    v_lshrrev_b16_e32 v3, 8, v1
; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v3, v0
; GFX9-DL-NEXT:    v_add_u32_e32 v1, v4, v5
; GFX9-DL-NEXT:    v_add3_u32 v0, v1, v0, v2
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: dot4_mixed_ext:
; GFX10-DL:       ; %bb.0: ; %entry
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_mov_b32_e32 v3, 0xffff
; GFX10-DL-NEXT:    v_lshrrev_b16 v4, 8, v1
; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v6, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX10-DL-NEXT:    v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v3, v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v5, v0
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v1, v3, v6
; GFX10-DL-NEXT:    v_add3_u32 v0, v0, v1, v2
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: dot4_mixed_ext:
; GFX950:       ; %bb.0: ; %entry
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    v_lshrrev_b16_e32 v3, 8, v1
; GFX950-NEXT:    v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX950-NEXT:    v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
; GFX950-NEXT:    v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX950-NEXT:    v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX950-NEXT:    v_add_u32_e32 v0, v3, v0
; GFX950-NEXT:    v_add_u32_e32 v1, v4, v5
; GFX950-NEXT:    v_add3_u32 v0, v1, v0, v2
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: dot4_mixed_ext:
; GFX9-NODL:       ; %bb.0: ; %entry
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_lshrrev_b16_e32 v3, 8, v1
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
; GFX9-NODL-NEXT:    v_add_u32_e32 v0, v3, v0
; GFX9-NODL-NEXT:    v_add_u32_e32 v1, v4, v5
; GFX9-NODL-NEXT:    v_add3_u32 v0, v1, v0, v2
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
entry:
  %a_vec = bitcast i32 %a_packed to <4 x i8>
  %b_vec = bitcast i32 %b_packed to <4 x i8>
  %a_ext = zext <4 x i8> %a_vec to <4 x i32>
  %b_ext = sext <4 x i8> %b_vec to <4 x i32>
  %mul = mul <4 x i32> %a_ext, %b_ext
  %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul)
  %result = add i32 %sum, %c
  ret i32 %result
}

;------------------------------------------------------------------------------
; NEGATIVE TESTS: CHAIN CONTAINS A NON-ADD NODE
;------------------------------------------------------------------------------

define i32 @dot4_and_chain_not_add(i32 %v) {
; GFX9-DL-LABEL: dot4_and_chain_not_add:
; GFX9-DL:       ; %bb.0:
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT:    v_and_b32_e32 v1, 1, v0
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v1, v0
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: dot4_and_chain_not_add:
; GFX10-DL:       ; %bb.0:
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT:    v_and_b32_e32 v1, 1, v0
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v1, v0
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: dot4_and_chain_not_add:
; GFX950:       ; %bb.0:
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    v_mul_u32_u24_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX950-NEXT:    v_and_b32_e32 v1, 1, v0
; GFX950-NEXT:    v_add_u32_e32 v0, v1, v0
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: dot4_and_chain_not_add:
; GFX9-NODL:       ; %bb.0:
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_and_b32_e32 v1, 1, v0
; GFX9-NODL-NEXT:    v_add_u32_e32 v0, v1, v0
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
  %masked = and i32 %v, 255
  %square = mul i32 %masked, %masked
  %lowbit = and i32 %square, 1
  %result = add i32 %lowbit, %square
  ret i32 %result
}

define i32 @dot4_or_chain_not_add(i32 %x, i32 %y, i32 %z) {
; GFX9-DL-LABEL: dot4_or_chain_not_add:
; GFX9-DL:       ; %bb.0:
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    v_mul_u32_u24_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT:    v_or_b32_e32 v1, 1, v1
; GFX9-DL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v0, v1
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: dot4_or_chain_not_add:
; GFX10-DL:       ; %bb.0:
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_mul_u32_u24_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT:    v_or_b32_e32 v1, 1, v1
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v0, v1
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: dot4_or_chain_not_add:
; GFX950:       ; %bb.0:
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    v_mul_u32_u24_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX950-NEXT:    v_or_b32_e32 v1, 1, v1
; GFX950-NEXT:    v_mul_u32_u24_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX950-NEXT:    v_add_u32_e32 v0, v0, v1
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: dot4_or_chain_not_add:
; GFX9-NODL:       ; %bb.0:
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_or_b32_e32 v1, 1, v1
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_add_u32_e32 v0, v0, v1
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
  %x.m = and i32 %x, 255
  %y.m = and i32 %y, 255
  %z.m = and i32 %z, 255
  %mul0 = mul i32 %x.m, %y.m
  %or = or i32 %mul0, 1
  %mul1 = mul i32 %x.m, %z.m
  %result = add i32 %mul1, %or
  ret i32 %result
}

; `or disjoint x, c` is equivalent to `add x, c`; this currently is not folded
; into the dot4 either because the matcher only accepts ISD::ADD. If that
; restriction is loosened, this becomes the positive case.
define i32 @dot4_or_disjoint_chain_not_add(i32 %x, i32 %y, i32 %z) {
; GFX9-DL-LABEL: dot4_or_disjoint_chain_not_add:
; GFX9-DL:       ; %bb.0:
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    v_and_b32_e32 v0, 0xfe, v0
; GFX9-DL-NEXT:    v_mul_u32_u24_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-DL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v0, v1
; GFX9-DL-NEXT:    v_or_b32_e32 v0, 1, v0
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: dot4_or_disjoint_chain_not_add:
; GFX10-DL:       ; %bb.0:
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_and_b32_e32 v0, 0xfe, v0
; GFX10-DL-NEXT:    v_mul_u32_u24_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX10-DL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v0, v1
; GFX10-DL-NEXT:    v_or_b32_e32 v0, 1, v0
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: dot4_or_disjoint_chain_not_add:
; GFX950:       ; %bb.0:
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    v_and_b32_e32 v0, 0xfe, v0
; GFX950-NEXT:    v_mul_u32_u24_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX950-NEXT:    v_mul_u32_u24_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX950-NEXT:    v_add_u32_e32 v0, v0, v1
; GFX950-NEXT:    v_or_b32_e32 v0, 1, v0
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: dot4_or_disjoint_chain_not_add:
; GFX9-NODL:       ; %bb.0:
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_and_b32_e32 v0, 0xfe, v0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_add_u32_e32 v0, v0, v1
; GFX9-NODL-NEXT:    v_or_b32_e32 v0, 1, v0
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
  %x.m = and i32 %x, 254
  %y.m = and i32 %y, 255
  %z.m = and i32 %z, 255
  %mul0 = mul i32 %x.m, %y.m
  %or = or disjoint i32 %mul0, 1
  %mul1 = mul i32 %x.m, %z.m
  %result = add i32 %mul1, %or
  ret i32 %result
}

; Same shape as the udot4 case, but signed. The matcher uses a shared loop, so
; the signed path must not regress either.
define i32 @sdot4_and_chain_not_add(i32 %v) {
; GFX9-DL-LABEL: sdot4_and_chain_not_add:
; GFX9-DL:       ; %bb.0:
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT:    v_and_b32_e32 v1, 1, v0
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v1, v0
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: sdot4_and_chain_not_add:
; GFX10-DL:       ; %bb.0:
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT:    v_and_b32_e32 v1, 1, v0
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v1, v0
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: sdot4_and_chain_not_add:
; GFX950:       ; %bb.0:
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX950-NEXT:    v_and_b32_e32 v1, 1, v0
; GFX950-NEXT:    v_add_u32_e32 v0, v1, v0
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: sdot4_and_chain_not_add:
; GFX9-NODL:       ; %bb.0:
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_and_b32_e32 v1, 1, v0
; GFX9-NODL-NEXT:    v_add_u32_e32 v0, v1, v0
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
  %shl = shl i32 %v, 24
  %byte = ashr i32 %shl, 24
  %square = mul i32 %byte, %byte
  %lowbit = and i32 %square, 1
  %result = add i32 %lowbit, %square
  ret i32 %result
}

; Before the chain-shape guard the addend walker stepped into the AND
; and pushed its 1023 mask as a dot4 accumulator, miscompiling the kernel
; to 0x3ff for an all-zero input.
define i32 @dot4_and_mask10_chain_not_add(i32 %x, i32 %y) {
; GFX9-DL-LABEL: dot4_and_mask10_chain_not_add:
; GFX9-DL:       ; %bb.0:
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
; GFX9-DL-NEXT:    v_add_u32_e32 v0, v1, v0
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: dot4_and_mask10_chain_not_add:
; GFX10-DL:       ; %bb.0:
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
; GFX10-DL-NEXT:    v_add_nc_u32_e32 v0, v1, v0
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: dot4_and_mask10_chain_not_add:
; GFX950:       ; %bb.0:
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX950-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
; GFX950-NEXT:    v_add_u32_e32 v0, v1, v0
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: dot4_and_mask10_chain_not_add:
; GFX9-NODL:       ; %bb.0:
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
; GFX9-NODL-NEXT:    v_add_u32_e32 v0, v1, v0
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
  %xb = and i32 %x, 255
  %yb = and i32 %y, 255
  %mul = mul i32 %xb, %yb
  %masked = and i32 %mul, 1023
  %result = add i32 %masked, %mul
  ret i32 %result
}

;------------------------------------------------------------------------------
; Negative tests: accumulator wider than a dot4 can represent
;------------------------------------------------------------------------------

; i64 accumulator would lose its upper half if folded into a 32-bit dot4.
define i64 @dot4_acc64_not_dot4(i8 %a0, i8 %a1, i8 %b0, i8 %b1, i64 %acc) {
; GFX9-DL-LABEL: dot4_acc64_not_dot4:
; GFX9-DL:       ; %bb.0:
; GFX9-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-DL-NEXT:    v_mul_hi_i32_i24_sdwa v6, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT:    v_mul_hi_i32_i24_sdwa v2, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT:    v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-DL-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v4
; GFX9-DL-NEXT:    v_addc_co_u32_e32 v2, vcc, v2, v5, vcc
; GFX9-DL-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v1
; GFX9-DL-NEXT:    v_addc_co_u32_e32 v1, vcc, v6, v2, vcc
; GFX9-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX10-DL-LABEL: dot4_acc64_not_dot4:
; GFX10-DL:       ; %bb.0:
; GFX10-DL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v6, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT:    v_mul_hi_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT:    v_mul_hi_i32_i24_sdwa v7, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX10-DL-NEXT:    v_add_co_u32 v2, vcc_lo, v6, v4
; GFX10-DL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v5, vcc_lo
; GFX10-DL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
; GFX10-DL-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v7, v1, vcc_lo
; GFX10-DL-NEXT:    s_setpc_b64 s[30:31]
;
; GFX950-LABEL: dot4_acc64_not_dot4:
; GFX950:       ; %bb.0:
; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT:    v_mul_hi_i32_i24_sdwa v9, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX950-NEXT:    v_mul_i32_i24_sdwa v8, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX950-NEXT:    v_mul_hi_i32_i24_sdwa v7, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX950-NEXT:    v_mul_i32_i24_sdwa v6, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX950-NEXT:    v_lshl_add_u64 v[0:1], v[8:9], 0, v[4:5]
; GFX950-NEXT:    v_lshl_add_u64 v[0:1], v[6:7], 0, v[0:1]
; GFX950-NEXT:    s_setpc_b64 s[30:31]
;
; GFX9-NODL-LABEL: dot4_acc64_not_dot4:
; GFX9-NODL:       ; %bb.0:
; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NODL-NEXT:    v_mul_hi_i32_i24_sdwa v6, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_hi_i32_i24_sdwa v2, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
; GFX9-NODL-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v4
; GFX9-NODL-NEXT:    v_addc_co_u32_e32 v2, vcc, v2, v5, vcc
; GFX9-NODL-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v1
; GFX9-NODL-NEXT:    v_addc_co_u32_e32 v1, vcc, v6, v2, vcc
; GFX9-NODL-NEXT:    s_setpc_b64 s[30:31]
  %cv1e0 = sext i8 %a0 to i64
  %cv1e1 = sext i8 %a1 to i64
  %cv2e0 = sext i8 %b0 to i64
  %cv2e1 = sext i8 %b1 to i64
  %mul0 = mul nsw i64 %cv1e0, %cv2e0
  %mul1 = mul nsw i64 %cv1e1, %cv2e1
  %add = add i64 %mul1, %acc
  %result = add i64 %mul0, %add
  ret i64 %result
}

declare i32 @llvm.sadd.sat.i32(i32, i32)
declare i32 @llvm.uadd.sat.i32(i32, i32)
declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)