| ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py |
| ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 < %s | FileCheck --check-prefixes=GFX9-DL %s |
| ; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 < %s | FileCheck --check-prefixes=GFX10-DL %s |
| ; RUN: llc -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GFX950 %s |
| ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck --check-prefixes=GFX9-NODL %s |
| ; RUN: opt -mtriple=amdgcn -mcpu=gfx906 -passes=amdgpu-codegenprepare -S < %s | FileCheck --check-prefixes=OPT %s |
| |
| ; Test dot4 patterns with saturating add (clamp) and without |
| |
| ;------------------------------------------------------------------------------ |
| ; DOT4 TESTS WITH <4 x i8> ARGS AND VECTOR REDUCE |
| ;------------------------------------------------------------------------------ |
| |
| define i32 @test_udot4_sat(<4 x i8> %a, <4 x i8> %b, i32 %c) { |
| ; GFX9-DL-LABEL: test_udot4_sat: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004 |
| ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4 |
| ; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX9-DL-NEXT: v_perm_b32 v1, v4, v5, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v2, v6, v7, s4 |
| ; GFX9-DL-NEXT: v_lshl_or_b32 v1, v2, 16, v1 |
| ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, v8 clamp |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: test_udot4_sat: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX10-DL-NEXT: v_lshl_or_b32 v1, v3, 16, v2 |
| ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, v8 clamp |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: test_udot4_sat: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004 |
| ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 |
| ; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0 |
| ; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX950-NEXT: v_perm_b32 v1, v4, v5, s0 |
| ; GFX950-NEXT: v_perm_b32 v2, v6, v7, s0 |
| ; GFX950-NEXT: v_lshl_or_b32 v1, v2, 16, v1 |
| ; GFX950-NEXT: v_dot4_u32_u8 v0, v1, v0, v8 clamp |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: test_udot4_sat: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v2, v1 |
| ; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v8 clamp |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| entry: |
| %conv.i = zext <4 x i8> %a to <4 x i32> |
| %conv8.i = zext <4 x i8> %b to <4 x i32> |
| %mul.i = mul <4 x i32> %conv8.i, %conv.i |
| %rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i) |
| %cond.i.i = tail call i32 @llvm.uadd.sat.i32(i32 %rdx.add.i, i32 %c) |
| ret i32 %cond.i.i |
| } |
| |
| define i32 @test_udot4_unsat(<4 x i8> %a, <4 x i8> %b, i32 %c) { |
| ; GFX9-DL-LABEL: test_udot4_unsat: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004 |
| ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4 |
| ; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX9-DL-NEXT: v_perm_b32 v1, v4, v5, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v2, v6, v7, s4 |
| ; GFX9-DL-NEXT: v_lshl_or_b32 v1, v2, 16, v1 |
| ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 0 |
| ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8 |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: test_udot4_unsat: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX10-DL-NEXT: v_lshl_or_b32 v1, v3, 16, v2 |
| ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 0 |
| ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v8 |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: test_udot4_unsat: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004 |
| ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 |
| ; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0 |
| ; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX950-NEXT: v_perm_b32 v1, v4, v5, s0 |
| ; GFX950-NEXT: v_perm_b32 v2, v6, v7, s0 |
| ; GFX950-NEXT: v_lshl_or_b32 v1, v2, 16, v1 |
| ; GFX950-NEXT: v_dot4_u32_u8 v0, v1, v0, 0 |
| ; GFX950-NEXT: s_nop 2 |
| ; GFX950-NEXT: v_add_u32_e32 v0, v0, v8 |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: test_udot4_unsat: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v0, v2 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v8 |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| entry: |
| %conv.i = zext <4 x i8> %a to <4 x i32> |
| %conv8.i = zext <4 x i8> %b to <4 x i32> |
| %mul.i = mul <4 x i32> %conv8.i, %conv.i |
| %rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i) |
| %add.i = add i32 %rdx.add.i, %c |
| ret i32 %add.i |
| } |
| |
| define i32 @test_sdot4_sat(<4 x i8> %a, <4 x i8> %b, i32 %c) { |
| ; GFX9-DL-LABEL: test_sdot4_sat: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004 |
| ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4 |
| ; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX9-DL-NEXT: v_perm_b32 v1, v4, v5, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v2, v6, v7, s4 |
| ; GFX9-DL-NEXT: v_lshl_or_b32 v1, v2, 16, v1 |
| ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v1, v0, v8 clamp |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: test_sdot4_sat: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX10-DL-NEXT: v_lshl_or_b32 v1, v3, 16, v2 |
| ; GFX10-DL-NEXT: v_dot4_i32_i8 v0, v1, v0, v8 clamp |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: test_sdot4_sat: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004 |
| ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 |
| ; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0 |
| ; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX950-NEXT: v_perm_b32 v1, v4, v5, s0 |
| ; GFX950-NEXT: v_perm_b32 v2, v6, v7, s0 |
| ; GFX950-NEXT: v_lshl_or_b32 v1, v2, 16, v1 |
| ; GFX950-NEXT: v_dot4_i32_i8 v0, v1, v0, v8 clamp |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: test_sdot4_sat: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v5), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v7), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v4), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v6), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v2, v1 |
| ; GFX9-NODL-NEXT: v_add_i32 v0, v0, v8 clamp |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| entry: |
| %conv.i = sext <4 x i8> %a to <4 x i32> |
| %conv8.i = sext <4 x i8> %b to <4 x i32> |
| %mul.i = mul <4 x i32> %conv8.i, %conv.i |
| %rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i) |
| %cond1.i.i = tail call i32 @llvm.sadd.sat.i32(i32 %rdx.add.i, i32 %c) |
| ret i32 %cond1.i.i |
| } |
| |
| define i32 @test_sdot4_unsat(<4 x i8> %a, <4 x i8> %b, i32 %c) { |
| ; GFX9-DL-LABEL: test_sdot4_unsat: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004 |
| ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4 |
| ; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX9-DL-NEXT: v_perm_b32 v1, v4, v5, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v2, v6, v7, s4 |
| ; GFX9-DL-NEXT: v_lshl_or_b32 v1, v2, 16, v1 |
| ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v1, v0, 0 |
| ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8 |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: test_sdot4_unsat: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX10-DL-NEXT: v_lshl_or_b32 v1, v3, 16, v2 |
| ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0 |
| ; GFX10-DL-NEXT: v_dot4c_i32_i8 v2, v1, v0 |
| ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v2, v8 |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: test_sdot4_unsat: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004 |
| ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 |
| ; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0 |
| ; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX950-NEXT: v_perm_b32 v1, v4, v5, s0 |
| ; GFX950-NEXT: v_perm_b32 v2, v6, v7, s0 |
| ; GFX950-NEXT: v_lshl_or_b32 v1, v2, 16, v1 |
| ; GFX950-NEXT: v_mov_b32_e32 v2, 0 |
| ; GFX950-NEXT: v_dot4c_i32_i8_e32 v2, v1, v0 |
| ; GFX950-NEXT: s_nop 2 |
| ; GFX950-NEXT: v_add_u32_e32 v0, v2, v8 |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: test_sdot4_unsat: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v4), sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v5), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v6), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v7), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v0, v2 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v8 |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| entry: |
| %conv.i = sext <4 x i8> %a to <4 x i32> |
| %conv8.i = sext <4 x i8> %b to <4 x i32> |
| %mul.i = mul <4 x i32> %conv8.i, %conv.i |
| %rdx.add.i = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul.i) |
| %add.i = add i32 %rdx.add.i, %c |
| ret i32 %add.i |
| } |
| |
| ;------------------------------------------------------------------------------ |
| ; DOT4 TESTS WITH BITCAST FROM I32 |
| ;------------------------------------------------------------------------------ |
| |
| ; Unsigned dot4 without saturation using bitcast from i32 |
| ; OPT-LABEL: @udot4_unsat_bitcast( |
| ; OPT: [[A:%.*]] = bitcast i32 %a_packed to <4 x i8> |
| ; OPT: [[B:%.*]] = bitcast i32 %b_packed to <4 x i8> |
| ; OPT: [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32 |
| ; OPT: [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32 |
| ; OPT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A32]], i32 [[B32]], i32 0, i1 false) |
| ; OPT: [[RES:%.*]] = add i32 [[DOT]], %c |
| ; OPT: ret i32 [[RES]] |
| ; |
| define i32 @udot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) { |
| ; GFX9-DL-LABEL: udot4_unsat_bitcast: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0 |
| ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v2 |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: udot4_unsat_bitcast: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0 |
| ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v2 |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: udot4_unsat_bitcast: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v1, 0 |
| ; GFX950-NEXT: s_nop 2 |
| ; GFX950-NEXT: v_add_u32_e32 v0, v0, v2 |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: udot4_unsat_bitcast: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0 |
| ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 |
| ; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v5, v6 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2 |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| entry: |
| %a_vec = bitcast i32 %a_packed to <4 x i8> |
| %b_vec = bitcast i32 %b_packed to <4 x i8> |
| %a_ext = zext <4 x i8> %a_vec to <4 x i32> |
| %b_ext = zext <4 x i8> %b_vec to <4 x i32> |
| %mul = mul <4 x i32> %a_ext, %b_ext |
| %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) |
| %result = add i32 %sum, %c |
| ret i32 %result |
| } |
| |
| ; Signed dot4 without saturation using bitcast from i32 |
| ; OPT-LABEL: @sdot4_unsat_bitcast( |
| ; OPT: [[A:%.*]] = bitcast i32 %a_packed to <4 x i8> |
| ; OPT: [[B:%.*]] = bitcast i32 %b_packed to <4 x i8> |
| ; OPT: [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32 |
| ; OPT: [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32 |
| ; OPT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A32]], i32 [[B32]], i32 0, i1 false) |
| ; OPT: [[RES:%.*]] = add i32 [[DOT]], %c |
| ; OPT: ret i32 [[RES]] |
| ; |
| define i32 @sdot4_unsat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) { |
| ; GFX9-DL-LABEL: sdot4_unsat_bitcast: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v1, 0 |
| ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v2 |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: sdot4_unsat_bitcast: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0 |
| ; GFX10-DL-NEXT: v_dot4c_i32_i8 v3, v0, v1 |
| ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v3, v2 |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: sdot4_unsat_bitcast: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: v_mov_b32_e32 v3, 0 |
| ; GFX950-NEXT: v_dot4c_i32_i8_e32 v3, v0, v1 |
| ; GFX950-NEXT: s_nop 2 |
| ; GFX950-NEXT: v_add_u32_e32 v0, v3, v2 |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: sdot4_unsat_bitcast: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 |
| ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v5, v4 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2 |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| entry: |
| %a_vec = bitcast i32 %a_packed to <4 x i8> |
| %b_vec = bitcast i32 %b_packed to <4 x i8> |
| %a_ext = sext <4 x i8> %a_vec to <4 x i32> |
| %b_ext = sext <4 x i8> %b_vec to <4 x i32> |
| %mul = mul <4 x i32> %a_ext, %b_ext |
| %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) |
| %result = add i32 %sum, %c |
| ret i32 %result |
| } |
| |
| ; Unsigned dot4 with saturation using bitcast from i32 |
| ; OPT-LABEL: @udot4_sat_bitcast( |
| ; OPT: [[A:%.*]] = bitcast i32 %a_packed to <4 x i8> |
| ; OPT: [[B:%.*]] = bitcast i32 %b_packed to <4 x i8> |
| ; OPT: [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32 |
| ; OPT: [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32 |
| ; OPT: [[DOT:%.*]] = call i32 @llvm.amdgcn.udot4(i32 [[A32]], i32 [[B32]], i32 %c, i1 true) |
| ; OPT: ret i32 [[DOT]] |
| ; |
| define i32 @udot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) { |
| ; GFX9-DL-LABEL: udot4_sat_bitcast: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, v2 clamp |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: udot4_sat_bitcast: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, v2 clamp |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: udot4_sat_bitcast: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v1, v2 clamp |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: udot4_sat_bitcast: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0 |
| ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 |
| ; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v6, v0 |
| ; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v2 clamp |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| entry: |
| %a_vec = bitcast i32 %a_packed to <4 x i8> |
| %b_vec = bitcast i32 %b_packed to <4 x i8> |
| %a_ext = zext <4 x i8> %a_vec to <4 x i32> |
| %b_ext = zext <4 x i8> %b_vec to <4 x i32> |
| %mul = mul <4 x i32> %a_ext, %b_ext |
| %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) |
| %result = call i32 @llvm.uadd.sat.i32(i32 %sum, i32 %c) |
| ret i32 %result |
| } |
| |
| ; Signed dot4 with saturation using bitcast from i32 |
| ; OPT-LABEL: @sdot4_sat_bitcast( |
| ; OPT: [[A:%.*]] = bitcast i32 %a_packed to <4 x i8> |
| ; OPT: [[B:%.*]] = bitcast i32 %b_packed to <4 x i8> |
| ; OPT: [[A32:%.*]] = bitcast <4 x i8> [[A]] to i32 |
| ; OPT: [[B32:%.*]] = bitcast <4 x i8> [[B]] to i32 |
| ; OPT: [[DOT:%.*]] = call i32 @llvm.amdgcn.sdot4(i32 [[A32]], i32 [[B32]], i32 %c, i1 true) |
| ; OPT: ret i32 [[DOT]] |
| ; |
| define i32 @sdot4_sat_bitcast(i32 %a_packed, i32 %b_packed, i32 %c) { |
| ; GFX9-DL-LABEL: sdot4_sat_bitcast: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v1, v2 clamp |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: sdot4_sat_bitcast: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_dot4_i32_i8 v0, v0, v1, v2 clamp |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: sdot4_sat_bitcast: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: v_dot4_i32_i8 v0, v0, v1, v2 clamp |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: sdot4_sat_bitcast: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 |
| ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v4, v0 |
| ; GFX9-NODL-NEXT: v_add_i32 v0, v0, v2 clamp |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| entry: |
| %a_vec = bitcast i32 %a_packed to <4 x i8> |
| %b_vec = bitcast i32 %b_packed to <4 x i8> |
| %a_ext = sext <4 x i8> %a_vec to <4 x i32> |
| %b_ext = sext <4 x i8> %b_vec to <4 x i32> |
| %mul = mul <4 x i32> %a_ext, %b_ext |
| %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) |
| %result = call i32 @llvm.sadd.sat.i32(i32 %sum, i32 %c) |
| ret i32 %result |
| } |
| |
| ;------------------------------------------------------------------------------ |
| ; TESTS WITH <4 x i8> FUNCTION ARGUMENTS |
| ;------------------------------------------------------------------------------ |
| |
| ; Test with raw <4 x i8> arguments (ABI passes as separate registers) |
| define i32 @udot4_v4i8_arg(<4 x i8> %a, <4 x i8> %b, i32 %c) { |
| ; GFX9-DL-LABEL: udot4_v4i8_arg: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004 |
| ; GFX9-DL-NEXT: v_perm_b32 v4, v4, v5, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v5, v6, v7, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4 |
| ; GFX9-DL-NEXT: v_lshl_or_b32 v4, v5, 16, v4 |
| ; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v4, 0 |
| ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8 |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: udot4_v4i8_arg: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_lshl_or_b32 v2, v5, 16, v4 |
| ; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, 0 |
| ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v8 |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: udot4_v4i8_arg: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004 |
| ; GFX950-NEXT: v_perm_b32 v4, v4, v5, s0 |
| ; GFX950-NEXT: v_perm_b32 v5, v6, v7, s0 |
| ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 |
| ; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0 |
| ; GFX950-NEXT: v_lshl_or_b32 v4, v5, 16, v4 |
| ; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v4, 0 |
| ; GFX950-NEXT: s_nop 2 |
| ; GFX950-NEXT: v_add_u32_e32 v0, v0, v8 |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: udot4_v4i8_arg: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v0, v2 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v8 |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| entry: |
| %a_ext = zext <4 x i8> %a to <4 x i32> |
| %b_ext = zext <4 x i8> %b to <4 x i32> |
| %mul = mul <4 x i32> %a_ext, %b_ext |
| %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) |
| %result = add i32 %sum, %c |
| ret i32 %result |
| } |
| |
| define i32 @sdot4_v4i8_arg(<4 x i8> %a, <4 x i8> %b, i32 %c) { |
| ; GFX9-DL-LABEL: sdot4_v4i8_arg: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0004 |
| ; GFX9-DL-NEXT: v_perm_b32 v4, v4, v5, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v5, v6, v7, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v3, s4 |
| ; GFX9-DL-NEXT: v_lshl_or_b32 v4, v5, 16, v4 |
| ; GFX9-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v4, 0 |
| ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8 |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: sdot4_v4i8_arg: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004 |
| ; GFX10-DL-NEXT: v_lshl_or_b32 v2, v5, 16, v4 |
| ; GFX10-DL-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 |
| ; GFX10-DL-NEXT: v_dot4c_i32_i8 v1, v0, v2 |
| ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v1, v8 |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: sdot4_v4i8_arg: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0004 |
| ; GFX950-NEXT: v_perm_b32 v4, v4, v5, s0 |
| ; GFX950-NEXT: v_perm_b32 v5, v6, v7, s0 |
| ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 |
| ; GFX950-NEXT: v_perm_b32 v1, v2, v3, s0 |
| ; GFX950-NEXT: v_lshl_or_b32 v4, v5, 16, v4 |
| ; GFX950-NEXT: v_lshl_or_b32 v0, v1, 16, v0 |
| ; GFX950-NEXT: v_mov_b32_e32 v1, 0 |
| ; GFX950-NEXT: v_dot4c_i32_i8_e32 v1, v0, v4 |
| ; GFX950-NEXT: s_nop 2 |
| ; GFX950-NEXT: v_add_u32_e32 v0, v1, v8 |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: sdot4_v4i8_arg: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v1, v3 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v0, v2 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v8 |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| entry: |
| %a_ext = sext <4 x i8> %a to <4 x i32> |
| %b_ext = sext <4 x i8> %b to <4 x i32> |
| %mul = mul <4 x i32> %a_ext, %b_ext |
| %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) |
| %result = add i32 %sum, %c |
| ret i32 %result |
| } |
| |
| ;------------------------------------------------------------------------------ |
| ; TESTS WITH I8 PROMOTION |
| ;------------------------------------------------------------------------------ |
| |
| ; Test unsigned dot4 with i8 values zext'd individually |
| define i32 @udot4_i8_zext(i8 %a0, i8 %a1, i8 %a2, i8 %a3, |
| ; GFX9-DL-LABEL: udot4_i8_zext: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0400 |
| ; GFX9-DL-NEXT: s_mov_b32 s5, 0x4000c0c |
| ; GFX9-DL-NEXT: v_perm_b32 v4, v4, v5, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v5, v7, v6, s5 |
| ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v1, v3, v2, s5 |
| ; GFX9-DL-NEXT: v_or_b32_e32 v4, v5, v4 |
| ; GFX9-DL-NEXT: v_or_b32_e32 v0, v1, v0 |
| ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v4, 0 |
| ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8 |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: udot4_i8_zext: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0400 |
| ; GFX10-DL-NEXT: v_perm_b32 v5, v7, v6, 0x4000c0c |
| ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0400 |
| ; GFX10-DL-NEXT: v_perm_b32 v1, v3, v2, 0x4000c0c |
| ; GFX10-DL-NEXT: v_or_b32_e32 v2, v5, v4 |
| ; GFX10-DL-NEXT: v_or_b32_e32 v0, v1, v0 |
| ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, 0 |
| ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v0, v8 |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: udot4_i8_zext: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0400 |
| ; GFX950-NEXT: s_mov_b32 s1, 0x4000c0c |
| ; GFX950-NEXT: v_perm_b32 v4, v4, v5, s0 |
| ; GFX950-NEXT: v_perm_b32 v5, v7, v6, s1 |
| ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 |
| ; GFX950-NEXT: v_perm_b32 v1, v3, v2, s1 |
| ; GFX950-NEXT: v_or_b32_e32 v4, v5, v4 |
| ; GFX950-NEXT: v_or_b32_e32 v0, v1, v0 |
| ; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v4, 0 |
| ; GFX950-NEXT: s_nop 2 |
| ; GFX950-NEXT: v_add_u32_e32 v0, v0, v8 |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: udot4_i8_zext: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v2 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v3, v8 |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| i8 %b0, i8 %b1, i8 %b2, i8 %b3, i32 %c) { |
| entry: |
| %a0_32 = zext i8 %a0 to i32 |
| %a1_32 = zext i8 %a1 to i32 |
| %a2_32 = zext i8 %a2 to i32 |
| %a3_32 = zext i8 %a3 to i32 |
| %b0_32 = zext i8 %b0 to i32 |
| %b1_32 = zext i8 %b1 to i32 |
| %b2_32 = zext i8 %b2 to i32 |
| %b3_32 = zext i8 %b3 to i32 |
| %m0 = mul i32 %a0_32, %b0_32 |
| %m1 = mul i32 %a1_32, %b1_32 |
| %m2 = mul i32 %a2_32, %b2_32 |
| %m3 = mul i32 %a3_32, %b3_32 |
| %sum01 = add i32 %m0, %m1 |
| %sum012 = add i32 %sum01, %m2 |
| %sum0123 = add i32 %sum012, %m3 |
| %result = add i32 %sum0123, %c |
| ret i32 %result |
| } |
| |
| ; Test signed dot4 with i8 values sext'd individually |
| define i32 @sdot4_i8_sext(i8 %a0, i8 %a1, i8 %a2, i8 %a3, |
| ; GFX9-DL-LABEL: sdot4_i8_sext: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0400 |
| ; GFX9-DL-NEXT: s_mov_b32 s5, 0x4000c0c |
| ; GFX9-DL-NEXT: v_perm_b32 v4, v4, v5, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v5, v7, v6, s5 |
| ; GFX9-DL-NEXT: v_perm_b32 v0, v0, v1, s4 |
| ; GFX9-DL-NEXT: v_perm_b32 v1, v3, v2, s5 |
| ; GFX9-DL-NEXT: v_or_b32_e32 v4, v5, v4 |
| ; GFX9-DL-NEXT: v_or_b32_e32 v0, v1, v0 |
| ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v4, 0 |
| ; GFX9-DL-NEXT: v_add_u32_e32 v0, v0, v8 |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: sdot4_i8_sext: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0400 |
| ; GFX10-DL-NEXT: v_perm_b32 v5, v7, v6, 0x4000c0c |
| ; GFX10-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0400 |
| ; GFX10-DL-NEXT: v_perm_b32 v1, v3, v2, 0x4000c0c |
| ; GFX10-DL-NEXT: v_or_b32_e32 v2, v5, v4 |
| ; GFX10-DL-NEXT: v_or_b32_e32 v0, v1, v0 |
| ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0 |
| ; GFX10-DL-NEXT: v_dot4c_i32_i8 v1, v0, v2 |
| ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v1, v8 |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: sdot4_i8_sext: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: s_mov_b32 s0, 0xc0c0400 |
| ; GFX950-NEXT: s_mov_b32 s1, 0x4000c0c |
| ; GFX950-NEXT: v_perm_b32 v4, v4, v5, s0 |
| ; GFX950-NEXT: v_perm_b32 v5, v7, v6, s1 |
| ; GFX950-NEXT: v_perm_b32 v0, v0, v1, s0 |
| ; GFX950-NEXT: v_perm_b32 v1, v3, v2, s1 |
| ; GFX950-NEXT: v_or_b32_e32 v4, v5, v4 |
| ; GFX950-NEXT: v_or_b32_e32 v0, v1, v0 |
| ; GFX950-NEXT: v_mov_b32_e32 v1, 0 |
| ; GFX950-NEXT: v_dot4c_i32_i8_e32 v1, v0, v4 |
| ; GFX950-NEXT: s_nop 2 |
| ; GFX950-NEXT: v_add_u32_e32 v0, v1, v8 |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: sdot4_i8_sext: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v1, v2 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v3, v8 |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| i8 %b0, i8 %b1, i8 %b2, i8 %b3, i32 %c) { |
| entry: |
| %a0_32 = sext i8 %a0 to i32 |
| %a1_32 = sext i8 %a1 to i32 |
| %a2_32 = sext i8 %a2 to i32 |
| %a3_32 = sext i8 %a3 to i32 |
| %b0_32 = sext i8 %b0 to i32 |
| %b1_32 = sext i8 %b1 to i32 |
| %b2_32 = sext i8 %b2 to i32 |
| %b3_32 = sext i8 %b3 to i32 |
| %m0 = mul i32 %a0_32, %b0_32 |
| %m1 = mul i32 %a1_32, %b1_32 |
| %m2 = mul i32 %a2_32, %b2_32 |
| %m3 = mul i32 %a3_32, %b3_32 |
| %sum01 = add i32 %m0, %m1 |
| %sum012 = add i32 %sum01, %m2 |
| %sum0123 = add i32 %sum012, %m3 |
| %result = add i32 %sum0123, %c |
| ret i32 %result |
| } |
| |
| ;------------------------------------------------------------------------------ |
| ; NEGATIVE TESTS |
| ;------------------------------------------------------------------------------ |
| |
| ; Negative test: mismatched signedness (zext with sadd.sat) |
| define i32 @dot4_mismatched_sign_zext_sadd(i32 %a_packed, i32 %b_packed, i32 %c) { |
| ; GFX9-DL-LABEL: dot4_mismatched_sign_zext_sadd: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0 |
| ; GFX9-DL-NEXT: v_add_i32 v0, v0, v2 clamp |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: dot4_mismatched_sign_zext_sadd: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0 |
| ; GFX10-DL-NEXT: v_add_nc_i32 v0, v0, v2 clamp |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: dot4_mismatched_sign_zext_sadd: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: v_dot4_u32_u8 v0, v0, v1, 0 |
| ; GFX950-NEXT: s_nop 2 |
| ; GFX950-NEXT: v_add_i32 v0, v0, v2 clamp |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: dot4_mismatched_sign_zext_sadd: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v0 |
| ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 |
| ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 |
| ; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v3, v4, v0 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v6, v0 |
| ; GFX9-NODL-NEXT: v_add_i32 v0, v0, v2 clamp |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| entry: |
| %a_vec = bitcast i32 %a_packed to <4 x i8> |
| %b_vec = bitcast i32 %b_packed to <4 x i8> |
| %a_ext = zext <4 x i8> %a_vec to <4 x i32> |
| %b_ext = zext <4 x i8> %b_vec to <4 x i32> |
| %mul = mul <4 x i32> %a_ext, %b_ext |
| %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) |
| %result = call i32 @llvm.sadd.sat.i32(i32 %sum, i32 %c) |
| ret i32 %result |
| } |
| |
| ; Negative test: mismatched signedness (sext with uadd.sat) |
| define i32 @dot4_mismatched_sign_sext_uadd(i32 %a_packed, i32 %b_packed, i32 %c) { |
| ; GFX9-DL-LABEL: dot4_mismatched_sign_sext_uadd: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v0, v1, 0 |
| ; GFX9-DL-NEXT: v_add_u32_e64 v0, v0, v2 clamp |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: dot4_mismatched_sign_sext_uadd: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0 |
| ; GFX10-DL-NEXT: v_dot4c_i32_i8 v3, v0, v1 |
| ; GFX10-DL-NEXT: v_add_nc_u32_e64 v0, v3, v2 clamp |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: dot4_mismatched_sign_sext_uadd: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: v_mov_b32_e32 v3, 0 |
| ; GFX950-NEXT: v_dot4c_i32_i8_e32 v3, v0, v1 |
| ; GFX950-NEXT: s_nop 2 |
| ; GFX950-NEXT: v_add_u32_e64 v0, v3, v2 clamp |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: dot4_mismatched_sign_sext_uadd: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 |
| ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v4, v0 |
| ; GFX9-NODL-NEXT: v_add_u32_e64 v0, v0, v2 clamp |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| entry: |
| %a_vec = bitcast i32 %a_packed to <4 x i8> |
| %b_vec = bitcast i32 %b_packed to <4 x i8> |
| %a_ext = sext <4 x i8> %a_vec to <4 x i32> |
| %b_ext = sext <4 x i8> %b_vec to <4 x i32> |
| %mul = mul <4 x i32> %a_ext, %b_ext |
| %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) |
| %result = call i32 @llvm.uadd.sat.i32(i32 %sum, i32 %c) |
| ret i32 %result |
| } |
| |
| ; Negative test: mixed zext/sext operands |
| define i32 @dot4_mixed_ext(i32 %a_packed, i32 %b_packed, i32 %c) { |
| ; GFX9-DL-LABEL: dot4_mixed_ext: |
| ; GFX9-DL: ; %bb.0: ; %entry |
| ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-DL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 |
| ; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0 |
| ; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 |
| ; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 |
| ; GFX9-DL-NEXT: v_add_u32_e32 v0, v3, v0 |
| ; GFX9-DL-NEXT: v_add_u32_e32 v1, v4, v5 |
| ; GFX9-DL-NEXT: v_add3_u32 v0, v1, v0, v2 |
| ; GFX9-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX10-DL-LABEL: dot4_mixed_ext: |
| ; GFX10-DL: ; %bb.0: ; %entry |
| ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0xffff |
| ; GFX10-DL-NEXT: v_lshrrev_b16 v4, 8, v1 |
| ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v6, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 |
| ; GFX10-DL-NEXT: v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1 |
| ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 |
| ; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v3, v3, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 |
| ; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, v5, v0 |
| ; GFX10-DL-NEXT: v_add_nc_u32_e32 v1, v3, v6 |
| ; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v2 |
| ; GFX10-DL-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX950-LABEL: dot4_mixed_ext: |
| ; GFX950: ; %bb.0: ; %entry |
| ; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX950-NEXT: v_lshrrev_b16_e32 v3, 8, v1 |
| ; GFX950-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX950-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0 |
| ; GFX950-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 |
| ; GFX950-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 |
| ; GFX950-NEXT: v_add_u32_e32 v0, v3, v0 |
| ; GFX950-NEXT: v_add_u32_e32 v1, v4, v5 |
| ; GFX950-NEXT: v_add3_u32 v0, v1, v0, v2 |
| ; GFX950-NEXT: s_setpc_b64 s[30:31] |
| ; |
| ; GFX9-NODL-LABEL: dot4_mixed_ext: |
| ; GFX9-NODL: ; %bb.0: ; %entry |
| ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) |
| ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_0 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 |
| ; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0 |
| ; GFX9-NODL-NEXT: v_add_u32_e32 v1, v4, v5 |
| ; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v0, v2 |
| ; GFX9-NODL-NEXT: s_setpc_b64 s[30:31] |
| entry: |
| %a_vec = bitcast i32 %a_packed to <4 x i8> |
| %b_vec = bitcast i32 %b_packed to <4 x i8> |
| %a_ext = zext <4 x i8> %a_vec to <4 x i32> |
| %b_ext = sext <4 x i8> %b_vec to <4 x i32> |
| %mul = mul <4 x i32> %a_ext, %b_ext |
| %sum = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %mul) |
| %result = add i32 %sum, %c |
| ret i32 %result |
| } |
| |
| declare i32 @llvm.sadd.sat.i32(i32, i32) |
| declare i32 @llvm.uadd.sat.i32(i32, i32) |
| declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>) |
