| ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py |
| ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -mattr=+avx512f -verify-machineinstrs | FileCheck %s |
| ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -mattr=+avx512f -verify-machineinstrs -enable-ipra | FileCheck -check-prefix=IPRA %s |
| |
| @buf = dso_local global [3072 x i8] zeroinitializer, align 64 |
| |
| define internal void @foo() { |
| ; CHECK-LABEL: foo: |
| ; CHECK: # %bb.0: # %entry |
| ; CHECK-NEXT: retq |
| ; |
| ; IPRA-LABEL: foo: |
| ; IPRA: # %bb.0: # %entry |
| ; IPRA-NEXT: retq |
| entry: |
| ret void |
| } |
| |
| define dso_local void @test_api(i16 signext %0, i16 signext %1) nounwind { |
| ; CHECK-LABEL: test_api: |
| ; CHECK: # %bb.0: |
| ; CHECK-NEXT: pushq %rbp |
| ; CHECK-NEXT: pushq %r15 |
| ; CHECK-NEXT: pushq %r14 |
| ; CHECK-NEXT: pushq %rbx |
| ; CHECK-NEXT: subq $2120, %rsp # imm = 0x848 |
| ; CHECK-NEXT: movl %esi, %ebx |
| ; CHECK-NEXT: movl %edi, %ebp |
| ; CHECK-NEXT: vpxord %zmm0, %zmm0, %zmm0 |
| ; CHECK-NEXT: vmovdqu64 %zmm0, (%rsp) |
| ; CHECK-NEXT: movb $1, (%rsp) |
| ; CHECK-NEXT: movw $8, {{[0-9]+}}(%rsp) |
| ; CHECK-NEXT: movb $8, {{[0-9]+}}(%rsp) |
| ; CHECK-NEXT: movw %bx, {{[0-9]+}}(%rsp) |
| ; CHECK-NEXT: movb %bpl, {{[0-9]+}}(%rsp) |
| ; CHECK-NEXT: movw %bx, {{[0-9]+}}(%rsp) |
| ; CHECK-NEXT: movb %bpl, {{[0-9]+}}(%rsp) |
| ; CHECK-NEXT: ldtilecfg (%rsp) |
| ; CHECK-NEXT: movl $buf, %eax |
| ; CHECK-NEXT: movl $32, %r14d |
| ; CHECK-NEXT: movw $8, %r15w |
| ; CHECK-NEXT: tileloadd (%rax,%r14), %tmm1 |
| ; CHECK-NEXT: movabsq $64, %rax |
| ; CHECK-NEXT: tilestored %tmm1, 1088(%rsp,%rax) # 1024-byte Folded Spill |
| ; CHECK-NEXT: movl $buf+1024, %eax |
| ; CHECK-NEXT: tileloadd (%rax,%r14), %tmm2 |
| ; CHECK-NEXT: movabsq $64, %rax |
| ; CHECK-NEXT: tilestored %tmm2, 64(%rsp,%rax) # 1024-byte Folded Spill |
| ; CHECK-NEXT: vzeroupper |
| ; CHECK-NEXT: callq foo |
| ; CHECK-NEXT: ldtilecfg (%rsp) |
| ; CHECK-NEXT: movl $buf+2048, %eax |
| ; CHECK-NEXT: tileloadd (%rax,%r14), %tmm0 |
| ; CHECK-NEXT: movabsq $64, %rcx |
| ; CHECK-NEXT: tileloadd 1088(%rsp,%rcx), %tmm1 # 1024-byte Folded Reload |
| ; CHECK-NEXT: movabsq $64, %rcx |
| ; CHECK-NEXT: tileloadd 64(%rsp,%rcx), %tmm2 # 1024-byte Folded Reload |
| ; CHECK-NEXT: tdpbssd %tmm2, %tmm1, %tmm0 |
| ; CHECK-NEXT: tilestored %tmm0, (%rax,%r14) |
| ; CHECK-NEXT: addq $2120, %rsp # imm = 0x848 |
| ; CHECK-NEXT: popq %rbx |
| ; CHECK-NEXT: popq %r14 |
| ; CHECK-NEXT: popq %r15 |
| ; CHECK-NEXT: popq %rbp |
| ; CHECK-NEXT: tilerelease |
| ; CHECK-NEXT: retq |
| ; |
| ; IPRA-LABEL: test_api: |
| ; IPRA: # %bb.0: |
| ; IPRA-NEXT: subq $72, %rsp |
| ; IPRA-NEXT: vpxord %zmm0, %zmm0, %zmm0 |
| ; IPRA-NEXT: vmovdqu64 %zmm0, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movb $1, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movw $8, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movb $8, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movw %si, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movb %dil, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movw %si, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movb %dil, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: ldtilecfg {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movl $buf, %eax |
| ; IPRA-NEXT: movl $32, %ecx |
| ; IPRA-NEXT: movw $8, %dx |
| ; IPRA-NEXT: tileloadd (%rax,%rcx), %tmm0 |
| ; IPRA-NEXT: movl $buf+1024, %eax |
| ; IPRA-NEXT: tileloadd (%rax,%rcx), %tmm1 |
| ; IPRA-NEXT: callq foo |
| ; IPRA-NEXT: movl $buf+2048, %eax |
| ; IPRA-NEXT: tileloadd (%rax,%rcx), %tmm2 |
| ; IPRA-NEXT: tdpbssd %tmm1, %tmm0, %tmm2 |
| ; IPRA-NEXT: tilestored %tmm2, (%rax,%rcx) |
| ; IPRA-NEXT: addq $72, %rsp |
| ; IPRA-NEXT: tilerelease |
| ; IPRA-NEXT: vzeroupper |
| ; IPRA-NEXT: retq |
| %3 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 %0, i16 8, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 0), i64 32) |
| %4 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 %1, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 1024), i64 32) |
| call void @foo() |
| %5 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 %0, i16 %1, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 2048), i64 32) |
| %6 = tail call x86_amx @llvm.x86.tdpbssd.internal(i16 %0, i16 %1, i16 8, x86_amx %5, x86_amx %3, x86_amx %4) |
| tail call void @llvm.x86.tilestored64.internal(i16 %0, i16 %1, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 2048), i64 32, x86_amx %6) |
| ret void |
| } |
| |
| define dso_local i32 @test_loop(i32 %0) nounwind { |
| ; CHECK-LABEL: test_loop: |
| ; CHECK: # %bb.0: |
| ; CHECK-NEXT: pushq %rbp |
| ; CHECK-NEXT: pushq %r15 |
| ; CHECK-NEXT: pushq %r14 |
| ; CHECK-NEXT: pushq %r13 |
| ; CHECK-NEXT: pushq %r12 |
| ; CHECK-NEXT: pushq %rbx |
| ; CHECK-NEXT: subq $1096, %rsp # imm = 0x448 |
| ; CHECK-NEXT: movl %edi, %r14d |
| ; CHECK-NEXT: vpxord %zmm0, %zmm0, %zmm0 |
| ; CHECK-NEXT: vmovdqu64 %zmm0, (%rsp) |
| ; CHECK-NEXT: movb $1, (%rsp) |
| ; CHECK-NEXT: movb $8, {{[0-9]+}}(%rsp) |
| ; CHECK-NEXT: movw $8, {{[0-9]+}}(%rsp) |
| ; CHECK-NEXT: vzeroupper |
| ; CHECK-NEXT: callq foo |
| ; CHECK-NEXT: ldtilecfg (%rsp) |
| ; CHECK-NEXT: testl %r14d, %r14d |
| ; CHECK-NEXT: jg .LBB2_4 |
| ; CHECK-NEXT: # %bb.1: # %.preheader |
| ; CHECK-NEXT: movl $7, %ebp |
| ; CHECK-NEXT: movl $buf, %r15d |
| ; CHECK-NEXT: movl $32, %r12d |
| ; CHECK-NEXT: movw $8, %bx |
| ; CHECK-NEXT: movl $buf+2048, %r13d |
| ; CHECK-NEXT: .p2align 4, 0x90 |
| ; CHECK-NEXT: .LBB2_2: # =>This Inner Loop Header: Depth=1 |
| ; CHECK-NEXT: tileloadd (%r15,%r12), %tmm0 |
| ; CHECK-NEXT: movabsq $64, %rax |
| ; CHECK-NEXT: tilestored %tmm0, 64(%rsp,%rax) # 1024-byte Folded Spill |
| ; CHECK-NEXT: callq foo |
| ; CHECK-NEXT: ldtilecfg (%rsp) |
| ; CHECK-NEXT: movabsq $64, %rax |
| ; CHECK-NEXT: tileloadd 64(%rsp,%rax), %tmm0 # 1024-byte Folded Reload |
| ; CHECK-NEXT: tilestored %tmm0, (%r13,%r12) |
| ; CHECK-NEXT: callq foo |
| ; CHECK-NEXT: ldtilecfg (%rsp) |
| ; CHECK-NEXT: decl %ebp |
| ; CHECK-NEXT: cmpl $7, %ebp |
| ; CHECK-NEXT: jne .LBB2_2 |
| ; CHECK-NEXT: # %bb.3: |
| ; CHECK-NEXT: cmpl $3, %r14d |
| ; CHECK-NEXT: jne .LBB2_4 |
| ; CHECK-NEXT: # %bb.6: |
| ; CHECK-NEXT: testl %ebp, %ebp |
| ; CHECK-NEXT: jne .LBB2_5 |
| ; CHECK-NEXT: # %bb.7: |
| ; CHECK-NEXT: incl %r14d |
| ; CHECK-NEXT: jmp .LBB2_8 |
| ; CHECK-NEXT: .LBB2_4: |
| ; CHECK-NEXT: callq foo |
| ; CHECK-NEXT: ldtilecfg (%rsp) |
| ; CHECK-NEXT: movl $32, %eax |
| ; CHECK-NEXT: movl $buf+1024, %ecx |
| ; CHECK-NEXT: movw $8, %dx |
| ; CHECK-NEXT: tileloadd (%rcx,%rax), %tmm0 |
| ; CHECK-NEXT: tilestored %tmm0, (%rcx,%rax) |
| ; CHECK-NEXT: .LBB2_5: |
| ; CHECK-NEXT: decl %r14d |
| ; CHECK-NEXT: .LBB2_8: |
| ; CHECK-NEXT: movl %r14d, %eax |
| ; CHECK-NEXT: addq $1096, %rsp # imm = 0x448 |
| ; CHECK-NEXT: popq %rbx |
| ; CHECK-NEXT: popq %r12 |
| ; CHECK-NEXT: popq %r13 |
| ; CHECK-NEXT: popq %r14 |
| ; CHECK-NEXT: popq %r15 |
| ; CHECK-NEXT: popq %rbp |
| ; CHECK-NEXT: tilerelease |
| ; CHECK-NEXT: retq |
| ; |
| ; IPRA-LABEL: test_loop: |
| ; IPRA: # %bb.0: |
| ; IPRA-NEXT: subq $72, %rsp |
| ; IPRA-NEXT: movl %edi, %eax |
| ; IPRA-NEXT: vpxord %zmm0, %zmm0, %zmm0 |
| ; IPRA-NEXT: vmovdqu64 %zmm0, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movb $1, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movb $8, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movw $8, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: ldtilecfg {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: callq foo |
| ; IPRA-NEXT: testl %edi, %edi |
| ; IPRA-NEXT: jg .LBB2_4 |
| ; IPRA-NEXT: # %bb.1: # %.preheader |
| ; IPRA-NEXT: movl $7, %ecx |
| ; IPRA-NEXT: movl $buf, %r8d |
| ; IPRA-NEXT: movl $32, %esi |
| ; IPRA-NEXT: movw $8, %di |
| ; IPRA-NEXT: movl $buf+2048, %edx |
| ; IPRA-NEXT: .p2align 4, 0x90 |
| ; IPRA-NEXT: .LBB2_2: # =>This Inner Loop Header: Depth=1 |
| ; IPRA-NEXT: tileloadd (%r8,%rsi), %tmm0 |
| ; IPRA-NEXT: callq foo |
| ; IPRA-NEXT: tilestored %tmm0, (%rdx,%rsi) |
| ; IPRA-NEXT: callq foo |
| ; IPRA-NEXT: decl %ecx |
| ; IPRA-NEXT: cmpl $7, %ecx |
| ; IPRA-NEXT: jne .LBB2_2 |
| ; IPRA-NEXT: # %bb.3: |
| ; IPRA-NEXT: cmpl $3, %eax |
| ; IPRA-NEXT: jne .LBB2_4 |
| ; IPRA-NEXT: # %bb.6: |
| ; IPRA-NEXT: testl %ecx, %ecx |
| ; IPRA-NEXT: jne .LBB2_5 |
| ; IPRA-NEXT: # %bb.7: |
| ; IPRA-NEXT: incl %eax |
| ; IPRA-NEXT: jmp .LBB2_8 |
| ; IPRA-NEXT: .LBB2_4: |
| ; IPRA-NEXT: callq foo |
| ; IPRA-NEXT: movl $32, %ecx |
| ; IPRA-NEXT: movl $buf+1024, %edx |
| ; IPRA-NEXT: movw $8, %si |
| ; IPRA-NEXT: tileloadd (%rdx,%rcx), %tmm0 |
| ; IPRA-NEXT: tilestored %tmm0, (%rdx,%rcx) |
| ; IPRA-NEXT: .LBB2_5: |
| ; IPRA-NEXT: decl %eax |
| ; IPRA-NEXT: .LBB2_8: |
| ; IPRA-NEXT: addq $72, %rsp |
| ; IPRA-NEXT: tilerelease |
| ; IPRA-NEXT: vzeroupper |
| ; IPRA-NEXT: retq |
| call void @foo() |
| br label %2 |
| 2: |
| %3 = icmp sgt i32 %0, 0 |
| br i1 %3, label %11, label %6 |
| 4: |
| %5 = icmp eq i32 %0, 3 |
| br i1 %5, label %13, label %11 |
| 6: |
| %7 = phi i32 [ %9, %6 ], [ 0, %2 ] |
| %8 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 8, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 0), i64 32) |
| call void @foo() |
| tail call void @llvm.x86.tilestored64.internal(i16 8, i16 8, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 2048), i64 32, x86_amx %8) |
| call void @foo() |
| %9 = add i32 %7, 1 |
| %10 = icmp eq i32 %9, 0 |
| br i1 %10, label %4, label %6 |
| 11: |
| call void @foo() |
| %12 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 8, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 1024), i64 32) |
| tail call void @llvm.x86.tilestored64.internal(i16 8, i16 8, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 1024), i64 32, x86_amx %12) |
| br label %17 |
| 13: |
| %14 = icmp eq i32 %9, 7 |
| br i1 %14, label %15, label %17 |
| 15: |
| %16 = add i32 %0, 1 |
| br label %19 |
| 17: |
| %18 = sub i32 %0, 1 |
| br label %19 |
| 19: |
| %20 = phi i32 [ %16, %15 ], [ %18, %17 ] |
| ret i32 %20 |
| } |
| |
| define dso_local void @test_loop2(i32 %0) nounwind { |
| ; CHECK-LABEL: test_loop2: |
| ; CHECK: # %bb.0: |
| ; CHECK-NEXT: pushq %rbp |
| ; CHECK-NEXT: pushq %r15 |
| ; CHECK-NEXT: pushq %r14 |
| ; CHECK-NEXT: pushq %r12 |
| ; CHECK-NEXT: pushq %rbx |
| ; CHECK-NEXT: subq $1088, %rsp # imm = 0x440 |
| ; CHECK-NEXT: movl %edi, %ebx |
| ; CHECK-NEXT: vpxord %zmm0, %zmm0, %zmm0 |
| ; CHECK-NEXT: vmovdqu64 %zmm0, (%rsp) |
| ; CHECK-NEXT: movb $1, (%rsp) |
| ; CHECK-NEXT: movb $8, {{[0-9]+}}(%rsp) |
| ; CHECK-NEXT: movw $8, {{[0-9]+}}(%rsp) |
| ; CHECK-NEXT: movl $buf, %r14d |
| ; CHECK-NEXT: movl $32, %r15d |
| ; CHECK-NEXT: movw $8, %bp |
| ; CHECK-NEXT: movl $buf+2048, %r12d |
| ; CHECK-NEXT: .p2align 4, 0x90 |
| ; CHECK-NEXT: .LBB3_1: # =>This Inner Loop Header: Depth=1 |
| ; CHECK-NEXT: vzeroupper |
| ; CHECK-NEXT: callq foo |
| ; CHECK-NEXT: ldtilecfg (%rsp) |
| ; CHECK-NEXT: testl %ebx, %ebx |
| ; CHECK-NEXT: jle .LBB3_3 |
| ; CHECK-NEXT: # %bb.2: # in Loop: Header=BB3_1 Depth=1 |
| ; CHECK-NEXT: tileloadd (%r14,%r15), %tmm0 |
| ; CHECK-NEXT: movabsq $64, %rax |
| ; CHECK-NEXT: tilestored %tmm0, 64(%rsp,%rax) # 1024-byte Folded Spill |
| ; CHECK-NEXT: callq foo |
| ; CHECK-NEXT: ldtilecfg (%rsp) |
| ; CHECK-NEXT: movabsq $64, %rax |
| ; CHECK-NEXT: tileloadd 64(%rsp,%rax), %tmm0 # 1024-byte Folded Reload |
| ; CHECK-NEXT: tilestored %tmm0, (%r12,%r15) |
| ; CHECK-NEXT: callq foo |
| ; CHECK-NEXT: jmp .LBB3_1 |
| ; CHECK-NEXT: .LBB3_3: |
| ; CHECK-NEXT: addq $1088, %rsp # imm = 0x440 |
| ; CHECK-NEXT: popq %rbx |
| ; CHECK-NEXT: popq %r12 |
| ; CHECK-NEXT: popq %r14 |
| ; CHECK-NEXT: popq %r15 |
| ; CHECK-NEXT: popq %rbp |
| ; CHECK-NEXT: tilerelease |
| ; CHECK-NEXT: retq |
| ; |
| ; IPRA-LABEL: test_loop2: |
| ; IPRA: # %bb.0: |
| ; IPRA-NEXT: subq $72, %rsp |
| ; IPRA-NEXT: vpxord %zmm0, %zmm0, %zmm0 |
| ; IPRA-NEXT: vmovdqu64 %zmm0, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movb $1, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movb $8, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movw $8, {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: ldtilecfg {{[0-9]+}}(%rsp) |
| ; IPRA-NEXT: movl $buf, %eax |
| ; IPRA-NEXT: movl $32, %ecx |
| ; IPRA-NEXT: movw $8, %dx |
| ; IPRA-NEXT: movl $buf+2048, %esi |
| ; IPRA-NEXT: .p2align 4, 0x90 |
| ; IPRA-NEXT: .LBB3_1: # =>This Inner Loop Header: Depth=1 |
| ; IPRA-NEXT: callq foo |
| ; IPRA-NEXT: testl %edi, %edi |
| ; IPRA-NEXT: jle .LBB3_3 |
| ; IPRA-NEXT: # %bb.2: # in Loop: Header=BB3_1 Depth=1 |
| ; IPRA-NEXT: tileloadd (%rax,%rcx), %tmm0 |
| ; IPRA-NEXT: callq foo |
| ; IPRA-NEXT: tilestored %tmm0, (%rsi,%rcx) |
| ; IPRA-NEXT: callq foo |
| ; IPRA-NEXT: jmp .LBB3_1 |
| ; IPRA-NEXT: .LBB3_3: |
| ; IPRA-NEXT: addq $72, %rsp |
| ; IPRA-NEXT: tilerelease |
| ; IPRA-NEXT: vzeroupper |
| ; IPRA-NEXT: retq |
| br label %2 |
| 2: |
| %3 = phi i32 [ 0, %1 ], [ %7, %5 ] |
| call void @foo() |
| %4 = icmp sgt i32 %0, 0 |
| br i1 %4, label %5, label %8 |
| 5: |
| %6 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 8, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 0), i64 32) |
| call void @foo() |
| tail call void @llvm.x86.tilestored64.internal(i16 8, i16 8, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 2048), i64 32, x86_amx %6) |
| call void @foo() |
| %7 = add i32 %3, 1 |
| br label %2 |
| 8: |
| ret void |
| } |
| |
| declare x86_amx @llvm.x86.tileloadd64.internal(i16, i16, i8*, i64) |
| declare x86_amx @llvm.x86.tdpbssd.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx) |
| declare void @llvm.x86.tilestored64.internal(i16, i16, i8*, i64, x86_amx) |
