| ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py |
| ; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s -check-prefixes=SSE,SSE2 |
| ; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s -check-prefixes=SSE,SSE42 |
| ; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s -check-prefixes=AVX,AVX2 |
| ; RUN: llc < %s -mtriple=x86_64-- -mcpu=knl | FileCheck %s -check-prefixes=AVX,AVX512,AVX512F |
| ; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s -check-prefixes=AVX,AVX512,AVX512VL |
| ; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 -mattr=+avx512vbmi2 | FileCheck %s -check-prefixes=AVX,AVX512,AVX512VBMI |
| |
| define i256 @fshl_i256(i256 %a0, i256 %a1, i256 %a2) nounwind { |
| ; SSE-LABEL: fshl_i256: |
| ; SSE: # %bb.0: |
| ; SSE-NEXT: pushq %rbx |
| ; SSE-NEXT: movq %rcx, %r10 |
| ; SSE-NEXT: movq %rdi, %rax |
| ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi |
| ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx |
| ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx |
| ; SSE-NEXT: testb $-128, %cl |
| ; SSE-NEXT: je .LBB0_1 |
| ; SSE-NEXT: # %bb.2: |
| ; SSE-NEXT: movq %rbx, %r11 |
| ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx |
| ; SSE-NEXT: jmp .LBB0_3 |
| ; SSE-NEXT: .LBB0_1: |
| ; SSE-NEXT: movq %rdi, %r9 |
| ; SSE-NEXT: movq %rdx, %r11 |
| ; SSE-NEXT: movq %rsi, %rdi |
| ; SSE-NEXT: movq %r10, %rsi |
| ; SSE-NEXT: movq %r8, %rdx |
| ; SSE-NEXT: .LBB0_3: |
| ; SSE-NEXT: testb $64, %cl |
| ; SSE-NEXT: cmovneq %rsi, %rdx |
| ; SSE-NEXT: cmovneq %r11, %rsi |
| ; SSE-NEXT: cmovneq %rdi, %r11 |
| ; SSE-NEXT: cmoveq %rbx, %r9 |
| ; SSE-NEXT: cmovneq %rbx, %rdi |
| ; SSE-NEXT: movq %rsi, %r8 |
| ; SSE-NEXT: shldq %cl, %r11, %r8 |
| ; SSE-NEXT: shldq %cl, %rsi, %rdx |
| ; SSE-NEXT: movq %rdi, %rsi |
| ; SSE-NEXT: shldq %cl, %r9, %rsi |
| ; SSE-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; SSE-NEXT: shldq %cl, %rdi, %r11 |
| ; SSE-NEXT: movq %rdx, 24(%rax) |
| ; SSE-NEXT: movq %r8, 16(%rax) |
| ; SSE-NEXT: movq %r11, 8(%rax) |
| ; SSE-NEXT: movq %rsi, (%rax) |
| ; SSE-NEXT: popq %rbx |
| ; SSE-NEXT: retq |
| ; |
| ; AVX-LABEL: fshl_i256: |
| ; AVX: # %bb.0: |
| ; AVX-NEXT: pushq %rbx |
| ; AVX-NEXT: movq %rcx, %r10 |
| ; AVX-NEXT: movq %rdi, %rax |
| ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rdi |
| ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rbx |
| ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rcx |
| ; AVX-NEXT: testb $-128, %cl |
| ; AVX-NEXT: je .LBB0_1 |
| ; AVX-NEXT: # %bb.2: |
| ; AVX-NEXT: movq %rbx, %r11 |
| ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rbx |
| ; AVX-NEXT: jmp .LBB0_3 |
| ; AVX-NEXT: .LBB0_1: |
| ; AVX-NEXT: movq %rdi, %r9 |
| ; AVX-NEXT: movq %rdx, %r11 |
| ; AVX-NEXT: movq %rsi, %rdi |
| ; AVX-NEXT: movq %r10, %rsi |
| ; AVX-NEXT: movq %r8, %rdx |
| ; AVX-NEXT: .LBB0_3: |
| ; AVX-NEXT: testb $64, %cl |
| ; AVX-NEXT: cmovneq %rsi, %rdx |
| ; AVX-NEXT: cmovneq %r11, %rsi |
| ; AVX-NEXT: cmovneq %rdi, %r11 |
| ; AVX-NEXT: cmoveq %rbx, %r9 |
| ; AVX-NEXT: cmovneq %rbx, %rdi |
| ; AVX-NEXT: movq %rsi, %r8 |
| ; AVX-NEXT: shldq %cl, %r11, %r8 |
| ; AVX-NEXT: shldq %cl, %rsi, %rdx |
| ; AVX-NEXT: movq %rdi, %rsi |
| ; AVX-NEXT: shldq %cl, %r9, %rsi |
| ; AVX-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; AVX-NEXT: shldq %cl, %rdi, %r11 |
| ; AVX-NEXT: movq %rdx, 24(%rax) |
| ; AVX-NEXT: movq %r8, 16(%rax) |
| ; AVX-NEXT: movq %r11, 8(%rax) |
| ; AVX-NEXT: movq %rsi, (%rax) |
| ; AVX-NEXT: popq %rbx |
| ; AVX-NEXT: retq |
| %r = call i256 @llvm.fshl.i256(i256 %a0, i256 %a1, i256 %a2) |
| ret i256 %r |
| } |
| |
| define i256 @fshr_i256(i256 %a0, i256 %a1, i256 %a2) nounwind { |
| ; SSE-LABEL: fshr_i256: |
| ; SSE: # %bb.0: |
| ; SSE-NEXT: pushq %rbx |
| ; SSE-NEXT: movq %rcx, %r10 |
| ; SSE-NEXT: movq %rdi, %rax |
| ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi |
| ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx |
| ; SSE-NEXT: movzbl {{[0-9]+}}(%rsp), %ecx |
| ; SSE-NEXT: testb %cl, %cl |
| ; SSE-NEXT: js .LBB1_1 |
| ; SSE-NEXT: # %bb.2: |
| ; SSE-NEXT: movq %rbx, %r11 |
| ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx |
| ; SSE-NEXT: jmp .LBB1_3 |
| ; SSE-NEXT: .LBB1_1: |
| ; SSE-NEXT: movq %rdi, %r9 |
| ; SSE-NEXT: movq %rsi, %rdi |
| ; SSE-NEXT: movq %rdx, %r11 |
| ; SSE-NEXT: movq %r10, %rsi |
| ; SSE-NEXT: movq %r8, %rdx |
| ; SSE-NEXT: .LBB1_3: |
| ; SSE-NEXT: testb $64, %cl |
| ; SSE-NEXT: cmoveq %rsi, %rdx |
| ; SSE-NEXT: cmoveq %r11, %rsi |
| ; SSE-NEXT: cmoveq %rdi, %r11 |
| ; SSE-NEXT: cmoveq %rbx, %rdi |
| ; SSE-NEXT: cmovneq %rbx, %r9 |
| ; SSE-NEXT: movq %r11, %r8 |
| ; SSE-NEXT: shrdq %cl, %rsi, %r8 |
| ; SSE-NEXT: shrdq %cl, %rdx, %rsi |
| ; SSE-NEXT: shrdq %cl, %rdi, %r9 |
| ; SSE-NEXT: shrdq %cl, %r11, %rdi |
| ; SSE-NEXT: movq %rsi, 24(%rax) |
| ; SSE-NEXT: movq %r8, 16(%rax) |
| ; SSE-NEXT: movq %rdi, 8(%rax) |
| ; SSE-NEXT: movq %r9, (%rax) |
| ; SSE-NEXT: popq %rbx |
| ; SSE-NEXT: retq |
| ; |
| ; AVX-LABEL: fshr_i256: |
| ; AVX: # %bb.0: |
| ; AVX-NEXT: pushq %rbx |
| ; AVX-NEXT: movq %rcx, %r10 |
| ; AVX-NEXT: movq %rdi, %rax |
| ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rdi |
| ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rbx |
| ; AVX-NEXT: movzbl {{[0-9]+}}(%rsp), %ecx |
| ; AVX-NEXT: testb %cl, %cl |
| ; AVX-NEXT: js .LBB1_1 |
| ; AVX-NEXT: # %bb.2: |
| ; AVX-NEXT: movq %rbx, %r11 |
| ; AVX-NEXT: movq {{[0-9]+}}(%rsp), %rbx |
| ; AVX-NEXT: jmp .LBB1_3 |
| ; AVX-NEXT: .LBB1_1: |
| ; AVX-NEXT: movq %rdi, %r9 |
| ; AVX-NEXT: movq %rsi, %rdi |
| ; AVX-NEXT: movq %rdx, %r11 |
| ; AVX-NEXT: movq %r10, %rsi |
| ; AVX-NEXT: movq %r8, %rdx |
| ; AVX-NEXT: .LBB1_3: |
| ; AVX-NEXT: testb $64, %cl |
| ; AVX-NEXT: cmoveq %rsi, %rdx |
| ; AVX-NEXT: cmoveq %r11, %rsi |
| ; AVX-NEXT: cmoveq %rdi, %r11 |
| ; AVX-NEXT: cmoveq %rbx, %rdi |
| ; AVX-NEXT: cmovneq %rbx, %r9 |
| ; AVX-NEXT: movq %r11, %r8 |
| ; AVX-NEXT: shrdq %cl, %rsi, %r8 |
| ; AVX-NEXT: shrdq %cl, %rdx, %rsi |
| ; AVX-NEXT: shrdq %cl, %rdi, %r9 |
| ; AVX-NEXT: shrdq %cl, %r11, %rdi |
| ; AVX-NEXT: movq %rsi, 24(%rax) |
| ; AVX-NEXT: movq %r8, 16(%rax) |
| ; AVX-NEXT: movq %rdi, 8(%rax) |
| ; AVX-NEXT: movq %r9, (%rax) |
| ; AVX-NEXT: popq %rbx |
| ; AVX-NEXT: retq |
| %r = call i256 @llvm.fshr.i256(i256 %a0, i256 %a1, i256 %a2) |
| ret i256 %r |
| } |
| |
| define i256 @fshl_rot_i256(i256 %a0, i256 %a1) nounwind { |
| ; SSE-LABEL: fshl_rot_i256: |
| ; SSE: # %bb.0: |
| ; SSE-NEXT: movq %rcx, %rax |
| ; SSE-NEXT: testb $-128, %r9b |
| ; SSE-NEXT: movq %r8, %rcx |
| ; SSE-NEXT: cmovneq %rdx, %rcx |
| ; SSE-NEXT: movq %rsi, %r10 |
| ; SSE-NEXT: cmovneq %rax, %r10 |
| ; SSE-NEXT: cmovneq %r8, %rdx |
| ; SSE-NEXT: cmovneq %rsi, %rax |
| ; SSE-NEXT: testb $64, %r9b |
| ; SSE-NEXT: movq %rax, %rsi |
| ; SSE-NEXT: cmovneq %rdx, %rsi |
| ; SSE-NEXT: cmovneq %r10, %rdx |
| ; SSE-NEXT: cmoveq %rcx, %rax |
| ; SSE-NEXT: cmovneq %rcx, %r10 |
| ; SSE-NEXT: movq %r10, %r8 |
| ; SSE-NEXT: movl %r9d, %ecx |
| ; SSE-NEXT: shldq %cl, %rax, %r8 |
| ; SSE-NEXT: movq %rdx, %r11 |
| ; SSE-NEXT: shldq %cl, %r10, %r11 |
| ; SSE-NEXT: movq %rsi, %r10 |
| ; SSE-NEXT: shldq %cl, %rdx, %r10 |
| ; SSE-NEXT: shldq %cl, %rsi, %rax |
| ; SSE-NEXT: movq %rax, 24(%rdi) |
| ; SSE-NEXT: movq %r10, 16(%rdi) |
| ; SSE-NEXT: movq %r11, 8(%rdi) |
| ; SSE-NEXT: movq %r8, (%rdi) |
| ; SSE-NEXT: movq %rdi, %rax |
| ; SSE-NEXT: retq |
| ; |
| ; AVX-LABEL: fshl_rot_i256: |
| ; AVX: # %bb.0: |
| ; AVX-NEXT: movq %rcx, %rax |
| ; AVX-NEXT: testb $-128, %r9b |
| ; AVX-NEXT: movq %r8, %rcx |
| ; AVX-NEXT: cmovneq %rdx, %rcx |
| ; AVX-NEXT: movq %rsi, %r10 |
| ; AVX-NEXT: cmovneq %rax, %r10 |
| ; AVX-NEXT: cmovneq %r8, %rdx |
| ; AVX-NEXT: cmovneq %rsi, %rax |
| ; AVX-NEXT: testb $64, %r9b |
| ; AVX-NEXT: movq %rax, %rsi |
| ; AVX-NEXT: cmovneq %rdx, %rsi |
| ; AVX-NEXT: cmovneq %r10, %rdx |
| ; AVX-NEXT: cmoveq %rcx, %rax |
| ; AVX-NEXT: cmovneq %rcx, %r10 |
| ; AVX-NEXT: movq %r10, %r8 |
| ; AVX-NEXT: movl %r9d, %ecx |
| ; AVX-NEXT: shldq %cl, %rax, %r8 |
| ; AVX-NEXT: movq %rdx, %r11 |
| ; AVX-NEXT: shldq %cl, %r10, %r11 |
| ; AVX-NEXT: movq %rsi, %r10 |
| ; AVX-NEXT: shldq %cl, %rdx, %r10 |
| ; AVX-NEXT: shldq %cl, %rsi, %rax |
| ; AVX-NEXT: movq %rax, 24(%rdi) |
| ; AVX-NEXT: movq %r10, 16(%rdi) |
| ; AVX-NEXT: movq %r11, 8(%rdi) |
| ; AVX-NEXT: movq %r8, (%rdi) |
| ; AVX-NEXT: movq %rdi, %rax |
| ; AVX-NEXT: retq |
| %r = call i256 @llvm.fshl.i256(i256 %a0, i256 %a0, i256 %a1) |
| ret i256 %r |
| } |
| |
| define i256 @fshr_rot_i256(i256 %a0, i256 %a1) nounwind { |
| ; SSE-LABEL: fshr_rot_i256: |
| ; SSE: # %bb.0: |
| ; SSE-NEXT: movq %rcx, %rax |
| ; SSE-NEXT: testb %r9b, %r9b |
| ; SSE-NEXT: movq %r8, %rcx |
| ; SSE-NEXT: cmovnsq %rdx, %rcx |
| ; SSE-NEXT: movq %rsi, %r10 |
| ; SSE-NEXT: cmovnsq %rax, %r10 |
| ; SSE-NEXT: cmovnsq %r8, %rdx |
| ; SSE-NEXT: cmovnsq %rsi, %rax |
| ; SSE-NEXT: testb $64, %r9b |
| ; SSE-NEXT: movq %rax, %rsi |
| ; SSE-NEXT: cmoveq %rdx, %rsi |
| ; SSE-NEXT: cmoveq %r10, %rdx |
| ; SSE-NEXT: cmoveq %rcx, %r10 |
| ; SSE-NEXT: cmovneq %rcx, %rax |
| ; SSE-NEXT: movq %rax, %r8 |
| ; SSE-NEXT: movl %r9d, %ecx |
| ; SSE-NEXT: shrdq %cl, %r10, %r8 |
| ; SSE-NEXT: shrdq %cl, %rdx, %r10 |
| ; SSE-NEXT: shrdq %cl, %rsi, %rdx |
| ; SSE-NEXT: shrdq %cl, %rax, %rsi |
| ; SSE-NEXT: movq %rdi, %rax |
| ; SSE-NEXT: movq %rsi, 24(%rdi) |
| ; SSE-NEXT: movq %rdx, 16(%rdi) |
| ; SSE-NEXT: movq %r10, 8(%rdi) |
| ; SSE-NEXT: movq %r8, (%rdi) |
| ; SSE-NEXT: retq |
| ; |
| ; AVX2-LABEL: fshr_rot_i256: |
| ; AVX2: # %bb.0: |
| ; AVX2-NEXT: movq %rcx, %rax |
| ; AVX2-NEXT: testb %r9b, %r9b |
| ; AVX2-NEXT: movq %r8, %rcx |
| ; AVX2-NEXT: cmovnsq %rdx, %rcx |
| ; AVX2-NEXT: movq %rsi, %r10 |
| ; AVX2-NEXT: cmovnsq %rax, %r10 |
| ; AVX2-NEXT: cmovnsq %r8, %rdx |
| ; AVX2-NEXT: cmovnsq %rsi, %rax |
| ; AVX2-NEXT: testb $64, %r9b |
| ; AVX2-NEXT: movq %rax, %rsi |
| ; AVX2-NEXT: cmoveq %rdx, %rsi |
| ; AVX2-NEXT: cmoveq %r10, %rdx |
| ; AVX2-NEXT: cmoveq %rcx, %r10 |
| ; AVX2-NEXT: cmovneq %rcx, %rax |
| ; AVX2-NEXT: movq %rax, %r8 |
| ; AVX2-NEXT: movl %r9d, %ecx |
| ; AVX2-NEXT: shrdq %cl, %r10, %r8 |
| ; AVX2-NEXT: shrdq %cl, %rdx, %r10 |
| ; AVX2-NEXT: shrdq %cl, %rsi, %rdx |
| ; AVX2-NEXT: shrdq %cl, %rax, %rsi |
| ; AVX2-NEXT: movq %rsi, 24(%rdi) |
| ; AVX2-NEXT: movq %rdx, 16(%rdi) |
| ; AVX2-NEXT: movq %r10, 8(%rdi) |
| ; AVX2-NEXT: movq %r8, (%rdi) |
| ; AVX2-NEXT: movq %rdi, %rax |
| ; AVX2-NEXT: retq |
| ; |
| ; AVX512F-LABEL: fshr_rot_i256: |
| ; AVX512F: # %bb.0: |
| ; AVX512F-NEXT: movq %rcx, %rax |
| ; AVX512F-NEXT: testb %r9b, %r9b |
| ; AVX512F-NEXT: movq %r8, %rcx |
| ; AVX512F-NEXT: cmovnsq %rdx, %rcx |
| ; AVX512F-NEXT: movq %rsi, %r10 |
| ; AVX512F-NEXT: cmovnsq %rax, %r10 |
| ; AVX512F-NEXT: cmovnsq %r8, %rdx |
| ; AVX512F-NEXT: cmovnsq %rsi, %rax |
| ; AVX512F-NEXT: testb $64, %r9b |
| ; AVX512F-NEXT: movq %rax, %rsi |
| ; AVX512F-NEXT: cmoveq %rdx, %rsi |
| ; AVX512F-NEXT: cmoveq %r10, %rdx |
| ; AVX512F-NEXT: cmoveq %rcx, %r10 |
| ; AVX512F-NEXT: cmovneq %rcx, %rax |
| ; AVX512F-NEXT: movq %rax, %r8 |
| ; AVX512F-NEXT: movl %r9d, %ecx |
| ; AVX512F-NEXT: shrdq %cl, %r10, %r8 |
| ; AVX512F-NEXT: shrdq %cl, %rdx, %r10 |
| ; AVX512F-NEXT: shrdq %cl, %rsi, %rdx |
| ; AVX512F-NEXT: shrdq %cl, %rax, %rsi |
| ; AVX512F-NEXT: movq %rsi, 24(%rdi) |
| ; AVX512F-NEXT: movq %rdx, 16(%rdi) |
| ; AVX512F-NEXT: movq %r10, 8(%rdi) |
| ; AVX512F-NEXT: movq %r8, (%rdi) |
| ; AVX512F-NEXT: movq %rdi, %rax |
| ; AVX512F-NEXT: retq |
| ; |
| ; AVX512VL-LABEL: fshr_rot_i256: |
| ; AVX512VL: # %bb.0: |
| ; AVX512VL-NEXT: movq %rcx, %rax |
| ; AVX512VL-NEXT: testb %r9b, %r9b |
| ; AVX512VL-NEXT: movq %r8, %rcx |
| ; AVX512VL-NEXT: cmovnsq %rdx, %rcx |
| ; AVX512VL-NEXT: movq %rsi, %r10 |
| ; AVX512VL-NEXT: cmovnsq %rax, %r10 |
| ; AVX512VL-NEXT: cmovnsq %r8, %rdx |
| ; AVX512VL-NEXT: cmovnsq %rsi, %rax |
| ; AVX512VL-NEXT: testb $64, %r9b |
| ; AVX512VL-NEXT: movq %rax, %rsi |
| ; AVX512VL-NEXT: cmoveq %rdx, %rsi |
| ; AVX512VL-NEXT: cmoveq %r10, %rdx |
| ; AVX512VL-NEXT: cmoveq %rcx, %r10 |
| ; AVX512VL-NEXT: cmovneq %rcx, %rax |
| ; AVX512VL-NEXT: movq %rax, %r8 |
| ; AVX512VL-NEXT: movl %r9d, %ecx |
| ; AVX512VL-NEXT: shrdq %cl, %r10, %r8 |
| ; AVX512VL-NEXT: shrdq %cl, %rdx, %r10 |
| ; AVX512VL-NEXT: shrdq %cl, %rsi, %rdx |
| ; AVX512VL-NEXT: shrdq %cl, %rax, %rsi |
| ; AVX512VL-NEXT: movq %rdi, %rax |
| ; AVX512VL-NEXT: movq %rsi, 24(%rdi) |
| ; AVX512VL-NEXT: movq %rdx, 16(%rdi) |
| ; AVX512VL-NEXT: movq %r10, 8(%rdi) |
| ; AVX512VL-NEXT: movq %r8, (%rdi) |
| ; AVX512VL-NEXT: retq |
| ; |
| ; AVX512VBMI-LABEL: fshr_rot_i256: |
| ; AVX512VBMI: # %bb.0: |
| ; AVX512VBMI-NEXT: movq %rcx, %rax |
| ; AVX512VBMI-NEXT: testb %r9b, %r9b |
| ; AVX512VBMI-NEXT: movq %r8, %rcx |
| ; AVX512VBMI-NEXT: cmovnsq %rdx, %rcx |
| ; AVX512VBMI-NEXT: movq %rsi, %r10 |
| ; AVX512VBMI-NEXT: cmovnsq %rax, %r10 |
| ; AVX512VBMI-NEXT: cmovnsq %r8, %rdx |
| ; AVX512VBMI-NEXT: cmovnsq %rsi, %rax |
| ; AVX512VBMI-NEXT: testb $64, %r9b |
| ; AVX512VBMI-NEXT: movq %rax, %rsi |
| ; AVX512VBMI-NEXT: cmoveq %rdx, %rsi |
| ; AVX512VBMI-NEXT: cmoveq %r10, %rdx |
| ; AVX512VBMI-NEXT: cmoveq %rcx, %r10 |
| ; AVX512VBMI-NEXT: cmovneq %rcx, %rax |
| ; AVX512VBMI-NEXT: movq %rax, %r8 |
| ; AVX512VBMI-NEXT: movl %r9d, %ecx |
| ; AVX512VBMI-NEXT: shrdq %cl, %r10, %r8 |
| ; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r10 |
| ; AVX512VBMI-NEXT: shrdq %cl, %rsi, %rdx |
| ; AVX512VBMI-NEXT: shrdq %cl, %rax, %rsi |
| ; AVX512VBMI-NEXT: movq %rdi, %rax |
| ; AVX512VBMI-NEXT: movq %rsi, 24(%rdi) |
| ; AVX512VBMI-NEXT: movq %rdx, 16(%rdi) |
| ; AVX512VBMI-NEXT: movq %r10, 8(%rdi) |
| ; AVX512VBMI-NEXT: movq %r8, (%rdi) |
| ; AVX512VBMI-NEXT: retq |
| %r = call i256 @llvm.fshr.i256(i256 %a0, i256 %a0, i256 %a1) |
| ret i256 %r |
| } |
| |
| define i256 @fshl_i256_load(ptr %p0, ptr %p1, i256 %a2) nounwind { |
| ; SSE-LABEL: fshl_i256_load: |
| ; SSE: # %bb.0: |
| ; SSE-NEXT: movq (%rsi), %rax |
| ; SSE-NEXT: movq 8(%rsi), %r8 |
| ; SSE-NEXT: movq 24(%rdx), %r9 |
| ; SSE-NEXT: testb $-128, %cl |
| ; SSE-NEXT: movq 8(%rdx), %r11 |
| ; SSE-NEXT: cmoveq %r9, %r11 |
| ; SSE-NEXT: movq 16(%rdx), %r10 |
| ; SSE-NEXT: movq (%rdx), %rdx |
| ; SSE-NEXT: cmoveq %r10, %rdx |
| ; SSE-NEXT: cmoveq %rax, %r10 |
| ; SSE-NEXT: cmoveq %r8, %r9 |
| ; SSE-NEXT: cmoveq 16(%rsi), %rax |
| ; SSE-NEXT: cmoveq 24(%rsi), %r8 |
| ; SSE-NEXT: testb $64, %cl |
| ; SSE-NEXT: cmovneq %rax, %r8 |
| ; SSE-NEXT: cmovneq %r9, %rax |
| ; SSE-NEXT: cmovneq %r10, %r9 |
| ; SSE-NEXT: cmoveq %r11, %rdx |
| ; SSE-NEXT: cmovneq %r11, %r10 |
| ; SSE-NEXT: movq %r10, %rsi |
| ; SSE-NEXT: shldq %cl, %rdx, %rsi |
| ; SSE-NEXT: movq %r9, %rdx |
| ; SSE-NEXT: shldq %cl, %r10, %rdx |
| ; SSE-NEXT: movq %rax, %r10 |
| ; SSE-NEXT: shldq %cl, %r9, %r10 |
| ; SSE-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; SSE-NEXT: shldq %cl, %rax, %r8 |
| ; SSE-NEXT: movq %rdi, %rax |
| ; SSE-NEXT: movq %r8, 24(%rdi) |
| ; SSE-NEXT: movq %r10, 16(%rdi) |
| ; SSE-NEXT: movq %rdx, 8(%rdi) |
| ; SSE-NEXT: movq %rsi, (%rdi) |
| ; SSE-NEXT: retq |
| ; |
| ; AVX2-LABEL: fshl_i256_load: |
| ; AVX2: # %bb.0: |
| ; AVX2-NEXT: pushq %rbx |
| ; AVX2-NEXT: movq (%rsi), %r9 |
| ; AVX2-NEXT: movq 8(%rsi), %r8 |
| ; AVX2-NEXT: movq 24(%rdx), %r10 |
| ; AVX2-NEXT: testb $-128, %cl |
| ; AVX2-NEXT: movq 8(%rdx), %rbx |
| ; AVX2-NEXT: cmoveq %r10, %rbx |
| ; AVX2-NEXT: movq 16(%rdx), %r11 |
| ; AVX2-NEXT: movq (%rdx), %rdx |
| ; AVX2-NEXT: cmoveq %r11, %rdx |
| ; AVX2-NEXT: cmoveq %r9, %r11 |
| ; AVX2-NEXT: cmoveq %r8, %r10 |
| ; AVX2-NEXT: cmoveq 16(%rsi), %r9 |
| ; AVX2-NEXT: cmoveq 24(%rsi), %r8 |
| ; AVX2-NEXT: movq %rdi, %rax |
| ; AVX2-NEXT: testb $64, %cl |
| ; AVX2-NEXT: cmovneq %r9, %r8 |
| ; AVX2-NEXT: cmovneq %r10, %r9 |
| ; AVX2-NEXT: cmovneq %r11, %r10 |
| ; AVX2-NEXT: cmoveq %rbx, %rdx |
| ; AVX2-NEXT: cmovneq %rbx, %r11 |
| ; AVX2-NEXT: movq %r11, %rsi |
| ; AVX2-NEXT: shldq %cl, %rdx, %rsi |
| ; AVX2-NEXT: movq %r10, %rdx |
| ; AVX2-NEXT: shldq %cl, %r11, %rdx |
| ; AVX2-NEXT: movq %r9, %rdi |
| ; AVX2-NEXT: shldq %cl, %r10, %rdi |
| ; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; AVX2-NEXT: shldq %cl, %r9, %r8 |
| ; AVX2-NEXT: movq %r8, 24(%rax) |
| ; AVX2-NEXT: movq %rdi, 16(%rax) |
| ; AVX2-NEXT: movq %rdx, 8(%rax) |
| ; AVX2-NEXT: movq %rsi, (%rax) |
| ; AVX2-NEXT: popq %rbx |
| ; AVX2-NEXT: retq |
| ; |
| ; AVX512F-LABEL: fshl_i256_load: |
| ; AVX512F: # %bb.0: |
| ; AVX512F-NEXT: movq %rdi, %rax |
| ; AVX512F-NEXT: vmovdqu (%rdx), %ymm0 |
| ; AVX512F-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm0 |
| ; AVX512F-NEXT: movzbl %cl, %ecx |
| ; AVX512F-NEXT: vmovq %rcx, %xmm1 |
| ; AVX512F-NEXT: # kill: def $ecx killed $ecx killed $rcx |
| ; AVX512F-NEXT: shrl $6, %ecx |
| ; AVX512F-NEXT: movl $-1, %edx |
| ; AVX512F-NEXT: shlxl %ecx, %edx, %ecx |
| ; AVX512F-NEXT: kmovw %ecx, %k1 |
| ; AVX512F-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512F-NEXT: vpbroadcastq %xmm1, %xmm1 |
| ; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm2 = [63,63] |
| ; AVX512F-NEXT: vpand %xmm2, %xmm1, %xmm3 |
| ; AVX512F-NEXT: vpsllq %xmm3, %zmm0, %zmm3 |
| ; AVX512F-NEXT: vextracti64x4 $1, %zmm3, %ymm3 |
| ; AVX512F-NEXT: valignq {{.*#+}} zmm0 = zmm0[7,0,1,2,3,4,5,6] |
| ; AVX512F-NEXT: vpsrlq $1, %zmm0, %zmm0 |
| ; AVX512F-NEXT: vpandn %xmm2, %xmm1, %xmm1 |
| ; AVX512F-NEXT: vpsrlq %xmm1, %zmm0, %zmm0 |
| ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0 |
| ; AVX512F-NEXT: vpor %ymm0, %ymm3, %ymm0 |
| ; AVX512F-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512F-NEXT: retq |
| ; |
| ; AVX512VL-LABEL: fshl_i256_load: |
| ; AVX512VL: # %bb.0: |
| ; AVX512VL-NEXT: movq %rdi, %rax |
| ; AVX512VL-NEXT: vmovdqu (%rdx), %ymm0 |
| ; AVX512VL-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm0 |
| ; AVX512VL-NEXT: movzbl %cl, %ecx |
| ; AVX512VL-NEXT: vpbroadcastq %rcx, %xmm1 |
| ; AVX512VL-NEXT: # kill: def $ecx killed $ecx killed $rcx |
| ; AVX512VL-NEXT: shrl $6, %ecx |
| ; AVX512VL-NEXT: movl $-1, %edx |
| ; AVX512VL-NEXT: shlxl %ecx, %edx, %ecx |
| ; AVX512VL-NEXT: kmovd %ecx, %k1 |
| ; AVX512VL-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm2 = [63,63] |
| ; AVX512VL-NEXT: vpand %xmm2, %xmm1, %xmm3 |
| ; AVX512VL-NEXT: vpsllq %xmm3, %zmm0, %zmm3 |
| ; AVX512VL-NEXT: vextracti64x4 $1, %zmm3, %ymm3 |
| ; AVX512VL-NEXT: valignq {{.*#+}} zmm0 = zmm0[7,0,1,2,3,4,5,6] |
| ; AVX512VL-NEXT: vpsrlq $1, %zmm0, %zmm0 |
| ; AVX512VL-NEXT: vpandn %xmm2, %xmm1, %xmm1 |
| ; AVX512VL-NEXT: vpsrlq %xmm1, %zmm0, %zmm0 |
| ; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 |
| ; AVX512VL-NEXT: vpor %ymm0, %ymm3, %ymm0 |
| ; AVX512VL-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512VL-NEXT: vzeroupper |
| ; AVX512VL-NEXT: retq |
| ; |
| ; AVX512VBMI-LABEL: fshl_i256_load: |
| ; AVX512VBMI: # %bb.0: |
| ; AVX512VBMI-NEXT: movq %rdi, %rax |
| ; AVX512VBMI-NEXT: vmovdqu (%rdx), %ymm0 |
| ; AVX512VBMI-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm0 |
| ; AVX512VBMI-NEXT: movzbl %cl, %ecx |
| ; AVX512VBMI-NEXT: movl %ecx, %edx |
| ; AVX512VBMI-NEXT: shrl $6, %edx |
| ; AVX512VBMI-NEXT: movl $-1, %esi |
| ; AVX512VBMI-NEXT: shlxl %edx, %esi, %edx |
| ; AVX512VBMI-NEXT: kmovd %edx, %k1 |
| ; AVX512VBMI-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm1 |
| ; AVX512VBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2 |
| ; AVX512VBMI-NEXT: valignq {{.*#+}} zmm2 = zmm2[7],zmm0[0,1,2,3,4,5,6] |
| ; AVX512VBMI-NEXT: vpshldvq %zmm1, %zmm2, %zmm0 |
| ; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm0, (%rdi) |
| ; AVX512VBMI-NEXT: vzeroupper |
| ; AVX512VBMI-NEXT: retq |
| %a0 = load i256, ptr %p0 |
| %a1 = load i256, ptr %p1 |
| %r = call i256 @llvm.fshl.i256(i256 %a0, i256 %a1, i256 %a2) |
| ret i256 %r |
| } |
| |
| define i256 @fshr_i256_load(ptr %p0, ptr %p1, i256 %a2) nounwind { |
| ; SSE-LABEL: fshr_i256_load: |
| ; SSE: # %bb.0: |
| ; SSE-NEXT: movq (%rsi), %r8 |
| ; SSE-NEXT: movq 8(%rsi), %rax |
| ; SSE-NEXT: movq 16(%rdx), %r9 |
| ; SSE-NEXT: testb %cl, %cl |
| ; SSE-NEXT: movq (%rdx), %r11 |
| ; SSE-NEXT: cmovsq %r9, %r11 |
| ; SSE-NEXT: movq 24(%rdx), %r10 |
| ; SSE-NEXT: movq 8(%rdx), %rdx |
| ; SSE-NEXT: cmovsq %r10, %rdx |
| ; SSE-NEXT: cmovsq %r8, %r9 |
| ; SSE-NEXT: cmovsq %rax, %r10 |
| ; SSE-NEXT: cmovsq 16(%rsi), %r8 |
| ; SSE-NEXT: cmovsq 24(%rsi), %rax |
| ; SSE-NEXT: testb $64, %cl |
| ; SSE-NEXT: cmoveq %r8, %rax |
| ; SSE-NEXT: cmoveq %r10, %r8 |
| ; SSE-NEXT: cmoveq %r9, %r10 |
| ; SSE-NEXT: cmoveq %rdx, %r9 |
| ; SSE-NEXT: cmoveq %r11, %rdx |
| ; SSE-NEXT: shrdq %cl, %r9, %rdx |
| ; SSE-NEXT: shrdq %cl, %r10, %r9 |
| ; SSE-NEXT: shrdq %cl, %r8, %r10 |
| ; SSE-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; SSE-NEXT: shrdq %cl, %rax, %r8 |
| ; SSE-NEXT: movq %rdi, %rax |
| ; SSE-NEXT: movq %r8, 24(%rdi) |
| ; SSE-NEXT: movq %r10, 16(%rdi) |
| ; SSE-NEXT: movq %r9, 8(%rdi) |
| ; SSE-NEXT: movq %rdx, (%rdi) |
| ; SSE-NEXT: retq |
| ; |
| ; AVX2-LABEL: fshr_i256_load: |
| ; AVX2: # %bb.0: |
| ; AVX2-NEXT: pushq %rbx |
| ; AVX2-NEXT: movq (%rsi), %r8 |
| ; AVX2-NEXT: movq 8(%rsi), %r10 |
| ; AVX2-NEXT: movq 16(%rdx), %r9 |
| ; AVX2-NEXT: testb %cl, %cl |
| ; AVX2-NEXT: movq (%rdx), %rbx |
| ; AVX2-NEXT: cmovsq %r9, %rbx |
| ; AVX2-NEXT: movq 24(%rdx), %r11 |
| ; AVX2-NEXT: movq 8(%rdx), %rdx |
| ; AVX2-NEXT: cmovsq %r11, %rdx |
| ; AVX2-NEXT: cmovsq %r8, %r9 |
| ; AVX2-NEXT: cmovsq %r10, %r11 |
| ; AVX2-NEXT: cmovsq 16(%rsi), %r8 |
| ; AVX2-NEXT: cmovsq 24(%rsi), %r10 |
| ; AVX2-NEXT: movq %rdi, %rax |
| ; AVX2-NEXT: testb $64, %cl |
| ; AVX2-NEXT: cmoveq %r8, %r10 |
| ; AVX2-NEXT: cmoveq %r11, %r8 |
| ; AVX2-NEXT: cmoveq %r9, %r11 |
| ; AVX2-NEXT: cmoveq %rdx, %r9 |
| ; AVX2-NEXT: cmoveq %rbx, %rdx |
| ; AVX2-NEXT: shrdq %cl, %r9, %rdx |
| ; AVX2-NEXT: shrdq %cl, %r11, %r9 |
| ; AVX2-NEXT: shrdq %cl, %r8, %r11 |
| ; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; AVX2-NEXT: shrdq %cl, %r10, %r8 |
| ; AVX2-NEXT: movq %r8, 24(%rdi) |
| ; AVX2-NEXT: movq %r11, 16(%rdi) |
| ; AVX2-NEXT: movq %r9, 8(%rdi) |
| ; AVX2-NEXT: movq %rdx, (%rdi) |
| ; AVX2-NEXT: popq %rbx |
| ; AVX2-NEXT: retq |
| ; |
| ; AVX512F-LABEL: fshr_i256_load: |
| ; AVX512F: # %bb.0: |
| ; AVX512F-NEXT: vmovdqu (%rdx), %ymm0 |
| ; AVX512F-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm0 |
| ; AVX512F-NEXT: movzbl %cl, %eax |
| ; AVX512F-NEXT: vmovq %rax, %xmm1 |
| ; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax |
| ; AVX512F-NEXT: shrl $6, %eax |
| ; AVX512F-NEXT: movl $-1, %ecx |
| ; AVX512F-NEXT: shlxl %eax, %ecx, %eax |
| ; AVX512F-NEXT: kmovw %eax, %k1 |
| ; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512F-NEXT: vpbroadcastq %xmm1, %xmm1 |
| ; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm2 = [63,63] |
| ; AVX512F-NEXT: vpand %xmm2, %xmm1, %xmm3 |
| ; AVX512F-NEXT: vpsrlq %xmm3, %ymm0, %ymm3 |
| ; AVX512F-NEXT: valignq {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7,0] |
| ; AVX512F-NEXT: vpaddq %ymm0, %ymm0, %ymm0 |
| ; AVX512F-NEXT: vpandn %xmm2, %xmm1, %xmm1 |
| ; AVX512F-NEXT: vpsllq %xmm1, %ymm0, %ymm0 |
| ; AVX512F-NEXT: movq %rdi, %rax |
| ; AVX512F-NEXT: vpor %ymm3, %ymm0, %ymm0 |
| ; AVX512F-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512F-NEXT: retq |
| ; |
| ; AVX512VL-LABEL: fshr_i256_load: |
| ; AVX512VL: # %bb.0: |
| ; AVX512VL-NEXT: movq %rdi, %rax |
| ; AVX512VL-NEXT: vmovdqu (%rdx), %ymm0 |
| ; AVX512VL-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm0 |
| ; AVX512VL-NEXT: movzbl %cl, %ecx |
| ; AVX512VL-NEXT: movl %ecx, %edx |
| ; AVX512VL-NEXT: shrl $6, %edx |
| ; AVX512VL-NEXT: movl $-1, %esi |
| ; AVX512VL-NEXT: shlxl %edx, %esi, %edx |
| ; AVX512VL-NEXT: kmovd %edx, %k1 |
| ; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm1 = [63,63] |
| ; AVX512VL-NEXT: vpbroadcastq %rcx, %xmm2 |
| ; AVX512VL-NEXT: vpand %xmm1, %xmm2, %xmm3 |
| ; AVX512VL-NEXT: vpsrlq %xmm3, %ymm0, %ymm3 |
| ; AVX512VL-NEXT: valignq {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7,0] |
| ; AVX512VL-NEXT: vpaddq %ymm0, %ymm0, %ymm0 |
| ; AVX512VL-NEXT: vpandn %xmm1, %xmm2, %xmm1 |
| ; AVX512VL-NEXT: vpsllq %xmm1, %ymm0, %ymm0 |
| ; AVX512VL-NEXT: vpor %ymm3, %ymm0, %ymm0 |
| ; AVX512VL-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512VL-NEXT: vzeroupper |
| ; AVX512VL-NEXT: retq |
| ; |
| ; AVX512VBMI-LABEL: fshr_i256_load: |
| ; AVX512VBMI: # %bb.0: |
| ; AVX512VBMI-NEXT: movq %rdi, %rax |
| ; AVX512VBMI-NEXT: vmovdqu (%rdx), %ymm0 |
| ; AVX512VBMI-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm0 |
| ; AVX512VBMI-NEXT: movzbl %cl, %ecx |
| ; AVX512VBMI-NEXT: movl %ecx, %edx |
| ; AVX512VBMI-NEXT: shrl $6, %edx |
| ; AVX512VBMI-NEXT: movl $-1, %esi |
| ; AVX512VBMI-NEXT: shlxl %edx, %esi, %edx |
| ; AVX512VBMI-NEXT: kmovd %edx, %k1 |
| ; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm1 |
| ; AVX512VBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2 |
| ; AVX512VBMI-NEXT: valignq {{.*#+}} zmm2 = zmm0[1,2,3,4,5,6,7],zmm2[0] |
| ; AVX512VBMI-NEXT: vpshrdvq %zmm1, %zmm2, %zmm0 |
| ; AVX512VBMI-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512VBMI-NEXT: vzeroupper |
| ; AVX512VBMI-NEXT: retq |
| %a0 = load i256, ptr %p0 |
| %a1 = load i256, ptr %p1 |
| %r = call i256 @llvm.fshr.i256(i256 %a0, i256 %a1, i256 %a2) |
| ret i256 %r |
| } |
| |
| define i256 @fshl_rot_i256_load(ptr %p0, i256 %a2) nounwind { |
| ; SSE-LABEL: fshl_rot_i256_load: |
| ; SSE: # %bb.0: |
| ; SSE-NEXT: movq %rdx, %rcx |
| ; SSE-NEXT: movq 16(%rsi), %rdx |
| ; SSE-NEXT: movq 8(%rsi), %rax |
| ; SSE-NEXT: movq 24(%rsi), %r8 |
| ; SSE-NEXT: testb $-128, %cl |
| ; SSE-NEXT: movq %r8, %r9 |
| ; SSE-NEXT: cmovneq %rax, %r9 |
| ; SSE-NEXT: movq (%rsi), %rsi |
| ; SSE-NEXT: movq %rsi, %r10 |
| ; SSE-NEXT: cmovneq %rdx, %r10 |
| ; SSE-NEXT: cmovneq %r8, %rax |
| ; SSE-NEXT: cmovneq %rsi, %rdx |
| ; SSE-NEXT: testb $64, %cl |
| ; SSE-NEXT: movq %rdx, %rsi |
| ; SSE-NEXT: cmovneq %rax, %rsi |
| ; SSE-NEXT: cmovneq %r10, %rax |
| ; SSE-NEXT: cmoveq %r9, %rdx |
| ; SSE-NEXT: cmovneq %r9, %r10 |
| ; SSE-NEXT: movq %r10, %r8 |
| ; SSE-NEXT: shldq %cl, %rdx, %r8 |
| ; SSE-NEXT: movq %rax, %r9 |
| ; SSE-NEXT: shldq %cl, %r10, %r9 |
| ; SSE-NEXT: movq %rsi, %r10 |
| ; SSE-NEXT: shldq %cl, %rax, %r10 |
| ; SSE-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; SSE-NEXT: shldq %cl, %rsi, %rdx |
| ; SSE-NEXT: movq %rdi, %rax |
| ; SSE-NEXT: movq %rdx, 24(%rdi) |
| ; SSE-NEXT: movq %r10, 16(%rdi) |
| ; SSE-NEXT: movq %r9, 8(%rdi) |
| ; SSE-NEXT: movq %r8, (%rdi) |
| ; SSE-NEXT: retq |
| ; |
| ; AVX2-LABEL: fshl_rot_i256_load: |
| ; AVX2: # %bb.0: |
| ; AVX2-NEXT: movq %rdx, %rcx |
| ; AVX2-NEXT: movq 16(%rsi), %rdx |
| ; AVX2-NEXT: movq (%rsi), %r9 |
| ; AVX2-NEXT: movq 8(%rsi), %r8 |
| ; AVX2-NEXT: movq 24(%rsi), %rax |
| ; AVX2-NEXT: testb $-128, %cl |
| ; AVX2-NEXT: movq %rax, %rsi |
| ; AVX2-NEXT: cmovneq %r8, %rsi |
| ; AVX2-NEXT: movq %r9, %r10 |
| ; AVX2-NEXT: cmovneq %rdx, %r10 |
| ; AVX2-NEXT: cmovneq %rax, %r8 |
| ; AVX2-NEXT: movq %rdi, %rax |
| ; AVX2-NEXT: cmovneq %r9, %rdx |
| ; AVX2-NEXT: testb $64, %cl |
| ; AVX2-NEXT: movq %rdx, %rdi |
| ; AVX2-NEXT: cmovneq %r8, %rdi |
| ; AVX2-NEXT: cmovneq %r10, %r8 |
| ; AVX2-NEXT: cmoveq %rsi, %rdx |
| ; AVX2-NEXT: cmovneq %rsi, %r10 |
| ; AVX2-NEXT: movq %r10, %rsi |
| ; AVX2-NEXT: shldq %cl, %rdx, %rsi |
| ; AVX2-NEXT: movq %r8, %r9 |
| ; AVX2-NEXT: shldq %cl, %r10, %r9 |
| ; AVX2-NEXT: movq %rdi, %r10 |
| ; AVX2-NEXT: shldq %cl, %r8, %r10 |
| ; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; AVX2-NEXT: shldq %cl, %rdi, %rdx |
| ; AVX2-NEXT: movq %rdx, 24(%rax) |
| ; AVX2-NEXT: movq %r10, 16(%rax) |
| ; AVX2-NEXT: movq %r9, 8(%rax) |
| ; AVX2-NEXT: movq %rsi, (%rax) |
| ; AVX2-NEXT: retq |
| ; |
| ; AVX512F-LABEL: fshl_rot_i256_load: |
| ; AVX512F: # %bb.0: |
| ; AVX512F-NEXT: movq %rdi, %rax |
| ; AVX512F-NEXT: movzbl %dl, %ecx |
| ; AVX512F-NEXT: vmovq %rcx, %xmm0 |
| ; AVX512F-NEXT: vpbroadcastq %xmm0, %xmm0 |
| ; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm1 = [63,63] |
| ; AVX512F-NEXT: vpandn %xmm1, %xmm0, %xmm2 |
| ; AVX512F-NEXT: shrl $6, %ecx |
| ; AVX512F-NEXT: movl $-1, %edx |
| ; AVX512F-NEXT: shlxl %ecx, %edx, %ecx |
| ; AVX512F-NEXT: kmovw %ecx, %k1 |
| ; AVX512F-NEXT: vbroadcasti64x4 {{.*#+}} zmm3 = mem[0,1,2,3,0,1,2,3] |
| ; AVX512F-NEXT: vpexpandq %zmm3, %zmm3 {%k1} {z} |
| ; AVX512F-NEXT: valignq {{.*#+}} zmm4 = zmm3[7,0,1,2,3,4,5,6] |
| ; AVX512F-NEXT: vpsrlq $1, %zmm4, %zmm4 |
| ; AVX512F-NEXT: vpsrlq %xmm2, %zmm4, %zmm2 |
| ; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm0 |
| ; AVX512F-NEXT: vpsllq %xmm0, %zmm3, %zmm0 |
| ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0 |
| ; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm1 |
| ; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 |
| ; AVX512F-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512F-NEXT: retq |
| ; |
| ; AVX512VL-LABEL: fshl_rot_i256_load: |
| ; AVX512VL: # %bb.0: |
| ; AVX512VL-NEXT: movzbl %dl, %eax |
| ; AVX512VL-NEXT: vpbroadcastq %rax, %xmm0 |
| ; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm1 = [63,63] |
| ; AVX512VL-NEXT: vpandn %xmm1, %xmm0, %xmm2 |
| ; AVX512VL-NEXT: shrl $6, %eax |
| ; AVX512VL-NEXT: movl $-1, %ecx |
| ; AVX512VL-NEXT: shlxl %eax, %ecx, %eax |
| ; AVX512VL-NEXT: kmovd %eax, %k1 |
| ; AVX512VL-NEXT: vbroadcasti64x4 {{.*#+}} zmm3 = mem[0,1,2,3,0,1,2,3] |
| ; AVX512VL-NEXT: vpexpandq %zmm3, %zmm3 {%k1} {z} |
| ; AVX512VL-NEXT: valignq {{.*#+}} zmm4 = zmm3[7,0,1,2,3,4,5,6] |
| ; AVX512VL-NEXT: vpsrlq $1, %zmm4, %zmm4 |
| ; AVX512VL-NEXT: vpsrlq %xmm2, %zmm4, %zmm2 |
| ; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm0 |
| ; AVX512VL-NEXT: vpsllq %xmm0, %zmm3, %zmm0 |
| ; AVX512VL-NEXT: movq %rdi, %rax |
| ; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 |
| ; AVX512VL-NEXT: vextracti64x4 $1, %zmm2, %ymm1 |
| ; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 |
| ; AVX512VL-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512VL-NEXT: vzeroupper |
| ; AVX512VL-NEXT: retq |
| ; |
| ; AVX512VBMI-LABEL: fshl_rot_i256_load: |
| ; AVX512VBMI: # %bb.0: |
| ; AVX512VBMI-NEXT: movq %rdi, %rax |
| ; AVX512VBMI-NEXT: movzbl %dl, %ecx |
| ; AVX512VBMI-NEXT: movl %ecx, %edx |
| ; AVX512VBMI-NEXT: shrl $6, %edx |
| ; AVX512VBMI-NEXT: movl $-1, %edi |
| ; AVX512VBMI-NEXT: shlxl %edx, %edi, %edx |
| ; AVX512VBMI-NEXT: kmovd %edx, %k1 |
| ; AVX512VBMI-NEXT: vbroadcasti64x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3] |
| ; AVX512VBMI-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm1 |
| ; AVX512VBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2 |
| ; AVX512VBMI-NEXT: valignq {{.*#+}} zmm2 = zmm2[7],zmm0[0,1,2,3,4,5,6] |
| ; AVX512VBMI-NEXT: vpshldvq %zmm1, %zmm2, %zmm0 |
| ; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm0, (%rax) |
| ; AVX512VBMI-NEXT: vzeroupper |
| ; AVX512VBMI-NEXT: retq |
| %a0 = load i256, ptr %p0 |
| %r = call i256 @llvm.fshl.i256(i256 %a0, i256 %a0, i256 %a2) |
| ret i256 %r |
| } |
| |
| define i256 @fshr_rot_i256_load(ptr %p0, i256 %a2) nounwind { |
| ; SSE-LABEL: fshr_rot_i256_load: |
| ; SSE: # %bb.0: |
| ; SSE-NEXT: movq %rdx, %rcx |
| ; SSE-NEXT: movq 16(%rsi), %rax |
| ; SSE-NEXT: movq 8(%rsi), %rdx |
| ; SSE-NEXT: movq 24(%rsi), %r8 |
| ; SSE-NEXT: testb %cl, %cl |
| ; SSE-NEXT: movq %r8, %r9 |
| ; SSE-NEXT: cmovnsq %rdx, %r9 |
| ; SSE-NEXT: movq (%rsi), %r10 |
| ; SSE-NEXT: movq %r10, %rsi |
| ; SSE-NEXT: cmovnsq %rax, %rsi |
| ; SSE-NEXT: cmovnsq %r8, %rdx |
| ; SSE-NEXT: cmovnsq %r10, %rax |
| ; SSE-NEXT: testb $64, %cl |
| ; SSE-NEXT: movq %rax, %r8 |
| ; SSE-NEXT: cmoveq %rdx, %r8 |
| ; SSE-NEXT: cmoveq %rsi, %rdx |
| ; SSE-NEXT: cmoveq %r9, %rsi |
| ; SSE-NEXT: cmovneq %r9, %rax |
| ; SSE-NEXT: movq %rax, %r9 |
| ; SSE-NEXT: shrdq %cl, %rsi, %r9 |
| ; SSE-NEXT: shrdq %cl, %rdx, %rsi |
| ; SSE-NEXT: shrdq %cl, %r8, %rdx |
| ; SSE-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; SSE-NEXT: shrdq %cl, %rax, %r8 |
| ; SSE-NEXT: movq %rdi, %rax |
| ; SSE-NEXT: movq %r8, 24(%rdi) |
| ; SSE-NEXT: movq %rdx, 16(%rdi) |
| ; SSE-NEXT: movq %rsi, 8(%rdi) |
| ; SSE-NEXT: movq %r9, (%rdi) |
| ; SSE-NEXT: retq |
| ; |
| ; AVX2-LABEL: fshr_rot_i256_load: |
| ; AVX2: # %bb.0: |
| ; AVX2-NEXT: movq %rdx, %rcx |
| ; AVX2-NEXT: movq 16(%rsi), %r8 |
| ; AVX2-NEXT: movq (%rsi), %r9 |
| ; AVX2-NEXT: movq 8(%rsi), %rdx |
| ; AVX2-NEXT: movq 24(%rsi), %rax |
| ; AVX2-NEXT: testb %cl, %cl |
| ; AVX2-NEXT: movq %rax, %r10 |
| ; AVX2-NEXT: cmovnsq %rdx, %r10 |
| ; AVX2-NEXT: movq %r9, %rsi |
| ; AVX2-NEXT: cmovnsq %r8, %rsi |
| ; AVX2-NEXT: cmovnsq %rax, %rdx |
| ; AVX2-NEXT: movq %rdi, %rax |
| ; AVX2-NEXT: cmovnsq %r9, %r8 |
| ; AVX2-NEXT: testb $64, %cl |
| ; AVX2-NEXT: movq %r8, %rdi |
| ; AVX2-NEXT: cmoveq %rdx, %rdi |
| ; AVX2-NEXT: cmoveq %rsi, %rdx |
| ; AVX2-NEXT: cmoveq %r10, %rsi |
| ; AVX2-NEXT: cmovneq %r10, %r8 |
| ; AVX2-NEXT: movq %r8, %r9 |
| ; AVX2-NEXT: shrdq %cl, %rsi, %r9 |
| ; AVX2-NEXT: shrdq %cl, %rdx, %rsi |
| ; AVX2-NEXT: shrdq %cl, %rdi, %rdx |
| ; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; AVX2-NEXT: shrdq %cl, %r8, %rdi |
| ; AVX2-NEXT: movq %rdi, 24(%rax) |
| ; AVX2-NEXT: movq %rdx, 16(%rax) |
| ; AVX2-NEXT: movq %rsi, 8(%rax) |
| ; AVX2-NEXT: movq %r9, (%rax) |
| ; AVX2-NEXT: retq |
| ; |
| ; AVX512F-LABEL: fshr_rot_i256_load: |
| ; AVX512F: # %bb.0: |
| ; AVX512F-NEXT: movzbl %dl, %eax |
| ; AVX512F-NEXT: vmovq %rax, %xmm0 |
| ; AVX512F-NEXT: vpbroadcastq %xmm0, %xmm0 |
| ; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm1 = [63,63] |
| ; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm2 |
| ; AVX512F-NEXT: shrl $6, %eax |
| ; AVX512F-NEXT: movl $-1, %ecx |
| ; AVX512F-NEXT: shlxl %eax, %ecx, %eax |
| ; AVX512F-NEXT: kmovw %eax, %k1 |
| ; AVX512F-NEXT: vbroadcasti64x4 {{.*#+}} zmm3 = mem[0,1,2,3,0,1,2,3] |
| ; AVX512F-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z} |
| ; AVX512F-NEXT: vpsrlq %xmm2, %ymm3, %ymm2 |
| ; AVX512F-NEXT: vpandn %xmm1, %xmm0, %xmm0 |
| ; AVX512F-NEXT: valignq {{.*#+}} zmm1 = zmm3[1,2,3,4,5,6,7,0] |
| ; AVX512F-NEXT: vpaddq %ymm1, %ymm1, %ymm1 |
| ; AVX512F-NEXT: vpsllq %xmm0, %ymm1, %ymm0 |
| ; AVX512F-NEXT: movq %rdi, %rax |
| ; AVX512F-NEXT: vpor %ymm2, %ymm0, %ymm0 |
| ; AVX512F-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512F-NEXT: retq |
| ; |
| ; AVX512VL-LABEL: fshr_rot_i256_load: |
| ; AVX512VL: # %bb.0: |
| ; AVX512VL-NEXT: movq %rdi, %rax |
| ; AVX512VL-NEXT: movzbl %dl, %ecx |
| ; AVX512VL-NEXT: vpbroadcastq %rcx, %xmm0 |
| ; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm1 = [63,63] |
| ; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm2 |
| ; AVX512VL-NEXT: shrl $6, %ecx |
| ; AVX512VL-NEXT: movl $-1, %edx |
| ; AVX512VL-NEXT: shlxl %ecx, %edx, %ecx |
| ; AVX512VL-NEXT: kmovd %ecx, %k1 |
| ; AVX512VL-NEXT: vbroadcasti64x4 {{.*#+}} zmm3 = mem[0,1,2,3,0,1,2,3] |
| ; AVX512VL-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z} |
| ; AVX512VL-NEXT: vpsrlq %xmm2, %ymm3, %ymm2 |
| ; AVX512VL-NEXT: vpandn %xmm1, %xmm0, %xmm0 |
| ; AVX512VL-NEXT: valignq {{.*#+}} zmm1 = zmm3[1,2,3,4,5,6,7,0] |
| ; AVX512VL-NEXT: vpaddq %ymm1, %ymm1, %ymm1 |
| ; AVX512VL-NEXT: vpsllq %xmm0, %ymm1, %ymm0 |
| ; AVX512VL-NEXT: vpor %ymm2, %ymm0, %ymm0 |
| ; AVX512VL-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512VL-NEXT: vzeroupper |
| ; AVX512VL-NEXT: retq |
| ; |
| ; AVX512VBMI-LABEL: fshr_rot_i256_load: |
| ; AVX512VBMI: # %bb.0: |
| ; AVX512VBMI-NEXT: movq %rdi, %rax |
| ; AVX512VBMI-NEXT: movzbl %dl, %ecx |
| ; AVX512VBMI-NEXT: movl %ecx, %edx |
| ; AVX512VBMI-NEXT: shrl $6, %edx |
| ; AVX512VBMI-NEXT: movl $-1, %edi |
| ; AVX512VBMI-NEXT: shlxl %edx, %edi, %edx |
| ; AVX512VBMI-NEXT: kmovd %edx, %k1 |
| ; AVX512VBMI-NEXT: vbroadcasti64x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3] |
| ; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm1 |
| ; AVX512VBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2 |
| ; AVX512VBMI-NEXT: valignq {{.*#+}} zmm2 = zmm0[1,2,3,4,5,6,7],zmm2[0] |
| ; AVX512VBMI-NEXT: vpshrdvq %zmm1, %zmm2, %zmm0 |
| ; AVX512VBMI-NEXT: vmovdqu %ymm0, (%rax) |
| ; AVX512VBMI-NEXT: vzeroupper |
| ; AVX512VBMI-NEXT: retq |
| %a0 = load i256, ptr %p0 |
| %r = call i256 @llvm.fshr.i256(i256 %a0, i256 %a0, i256 %a2) |
| ret i256 %r |
| } |
| |
| define i256 @fshl_i256_vector(<4 x i64> %v0, <4 x i64> %v1, i256 %a2) nounwind { |
| ; SSE2-LABEL: fshl_i256_vector: |
| ; SSE2: # %bb.0: |
| ; SSE2-NEXT: movq %rsi, %rcx |
| ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3] |
| ; SSE2-NEXT: movq %xmm1, %rax |
| ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] |
| ; SSE2-NEXT: movq %xmm1, %r10 |
| ; SSE2-NEXT: movq %xmm0, %r9 |
| ; SSE2-NEXT: movq %xmm2, %r8 |
| ; SSE2-NEXT: movq %xmm3, %rsi |
| ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3] |
| ; SSE2-NEXT: movq %xmm0, %r11 |
| ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3] |
| ; SSE2-NEXT: movq %xmm0, %rdx |
| ; SSE2-NEXT: testb $-128, %cl |
| ; SSE2-NEXT: cmoveq %rdx, %r11 |
| ; SSE2-NEXT: cmoveq %rsi, %r8 |
| ; SSE2-NEXT: cmoveq %r9, %rsi |
| ; SSE2-NEXT: cmoveq %r10, %rdx |
| ; SSE2-NEXT: cmovneq %r9, %rax |
| ; SSE2-NEXT: movq %xmm4, %r9 |
| ; SSE2-NEXT: cmovneq %r10, %r9 |
| ; SSE2-NEXT: testb $64, %cl |
| ; SSE2-NEXT: cmovneq %rax, %r9 |
| ; SSE2-NEXT: cmovneq %rdx, %rax |
| ; SSE2-NEXT: cmovneq %rsi, %rdx |
| ; SSE2-NEXT: cmoveq %r11, %r8 |
| ; SSE2-NEXT: cmovneq %r11, %rsi |
| ; SSE2-NEXT: movq %rsi, %r10 |
| ; SSE2-NEXT: shldq %cl, %r8, %r10 |
| ; SSE2-NEXT: movq %rdx, %r8 |
| ; SSE2-NEXT: shldq %cl, %rsi, %r8 |
| ; SSE2-NEXT: movq %rax, %rsi |
| ; SSE2-NEXT: shldq %cl, %rdx, %rsi |
| ; SSE2-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; SSE2-NEXT: shldq %cl, %rax, %r9 |
| ; SSE2-NEXT: movq %rdi, %rax |
| ; SSE2-NEXT: movq %r9, 24(%rdi) |
| ; SSE2-NEXT: movq %rsi, 16(%rdi) |
| ; SSE2-NEXT: movq %r8, 8(%rdi) |
| ; SSE2-NEXT: movq %r10, (%rdi) |
| ; SSE2-NEXT: retq |
| ; |
| ; SSE42-LABEL: fshl_i256_vector: |
| ; SSE42: # %bb.0: |
| ; SSE42-NEXT: pushq %rbx |
| ; SSE42-NEXT: movq %rsi, %rcx |
| ; SSE42-NEXT: pextrq $1, %xmm1, %rdx |
| ; SSE42-NEXT: movq %xmm1, %rax |
| ; SSE42-NEXT: pextrq $1, %xmm0, %r10 |
| ; SSE42-NEXT: movq %xmm0, %r11 |
| ; SSE42-NEXT: movq %xmm2, %r8 |
| ; SSE42-NEXT: pextrq $1, %xmm2, %rbx |
| ; SSE42-NEXT: pextrq $1, %xmm3, %rsi |
| ; SSE42-NEXT: movq %xmm3, %r9 |
| ; SSE42-NEXT: testb $-128, %cl |
| ; SSE42-NEXT: cmoveq %rsi, %rbx |
| ; SSE42-NEXT: cmoveq %r9, %r8 |
| ; SSE42-NEXT: cmoveq %r11, %r9 |
| ; SSE42-NEXT: cmoveq %r10, %rsi |
| ; SSE42-NEXT: cmovneq %r11, %rax |
| ; SSE42-NEXT: cmovneq %r10, %rdx |
| ; SSE42-NEXT: testb $64, %cl |
| ; SSE42-NEXT: cmovneq %rax, %rdx |
| ; SSE42-NEXT: cmovneq %rsi, %rax |
| ; SSE42-NEXT: cmovneq %r9, %rsi |
| ; SSE42-NEXT: cmoveq %rbx, %r8 |
| ; SSE42-NEXT: cmovneq %rbx, %r9 |
| ; SSE42-NEXT: movq %r9, %r10 |
| ; SSE42-NEXT: shldq %cl, %r8, %r10 |
| ; SSE42-NEXT: movq %rsi, %r8 |
| ; SSE42-NEXT: shldq %cl, %r9, %r8 |
| ; SSE42-NEXT: movq %rax, %r9 |
| ; SSE42-NEXT: shldq %cl, %rsi, %r9 |
| ; SSE42-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; SSE42-NEXT: shldq %cl, %rax, %rdx |
| ; SSE42-NEXT: movq %rdi, %rax |
| ; SSE42-NEXT: movq %rdx, 24(%rdi) |
| ; SSE42-NEXT: movq %r9, 16(%rdi) |
| ; SSE42-NEXT: movq %r8, 8(%rdi) |
| ; SSE42-NEXT: movq %r10, (%rdi) |
| ; SSE42-NEXT: popq %rbx |
| ; SSE42-NEXT: retq |
| ; |
| ; AVX2-LABEL: fshl_i256_vector: |
| ; AVX2: # %bb.0: |
| ; AVX2-NEXT: pushq %rbx |
| ; AVX2-NEXT: movq %rsi, %rcx |
| ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 |
| ; AVX2-NEXT: vpextrq $1, %xmm2, %rdx |
| ; AVX2-NEXT: vmovq %xmm2, %rsi |
| ; AVX2-NEXT: vpextrq $1, %xmm0, %rax |
| ; AVX2-NEXT: vmovq %xmm0, %r11 |
| ; AVX2-NEXT: vmovq %xmm1, %r9 |
| ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm0 |
| ; AVX2-NEXT: vpextrq $1, %xmm1, %rbx |
| ; AVX2-NEXT: vmovq %xmm0, %r10 |
| ; AVX2-NEXT: vpextrq $1, %xmm0, %r8 |
| ; AVX2-NEXT: testb $-128, %cl |
| ; AVX2-NEXT: cmoveq %r8, %rbx |
| ; AVX2-NEXT: cmoveq %r10, %r9 |
| ; AVX2-NEXT: cmoveq %r11, %r10 |
| ; AVX2-NEXT: cmoveq %rax, %r8 |
| ; AVX2-NEXT: cmovneq %r11, %rsi |
| ; AVX2-NEXT: cmovneq %rax, %rdx |
| ; AVX2-NEXT: movq %rdi, %rax |
| ; AVX2-NEXT: testb $64, %cl |
| ; AVX2-NEXT: cmovneq %rsi, %rdx |
| ; AVX2-NEXT: cmovneq %r8, %rsi |
| ; AVX2-NEXT: cmovneq %r10, %r8 |
| ; AVX2-NEXT: cmoveq %rbx, %r9 |
| ; AVX2-NEXT: cmovneq %rbx, %r10 |
| ; AVX2-NEXT: movq %r10, %rdi |
| ; AVX2-NEXT: shldq %cl, %r9, %rdi |
| ; AVX2-NEXT: movq %r8, %r9 |
| ; AVX2-NEXT: shldq %cl, %r10, %r9 |
| ; AVX2-NEXT: movq %rsi, %r10 |
| ; AVX2-NEXT: shldq %cl, %r8, %r10 |
| ; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; AVX2-NEXT: shldq %cl, %rsi, %rdx |
| ; AVX2-NEXT: movq %rdx, 24(%rax) |
| ; AVX2-NEXT: movq %r10, 16(%rax) |
| ; AVX2-NEXT: movq %r9, 8(%rax) |
| ; AVX2-NEXT: movq %rdi, (%rax) |
| ; AVX2-NEXT: popq %rbx |
| ; AVX2-NEXT: vzeroupper |
| ; AVX2-NEXT: retq |
| ; |
| ; AVX512F-LABEL: fshl_i256_vector: |
| ; AVX512F: # %bb.0: |
| ; AVX512F-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 |
| ; AVX512F-NEXT: movq %rdi, %rax |
| ; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 |
| ; AVX512F-NEXT: movzbl %sil, %ecx |
| ; AVX512F-NEXT: vmovq %rcx, %xmm1 |
| ; AVX512F-NEXT: # kill: def $ecx killed $ecx killed $rcx |
| ; AVX512F-NEXT: shrl $6, %ecx |
| ; AVX512F-NEXT: movl $-1, %edx |
| ; AVX512F-NEXT: shlxl %ecx, %edx, %ecx |
| ; AVX512F-NEXT: kmovw %ecx, %k1 |
| ; AVX512F-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512F-NEXT: vpbroadcastq %xmm1, %xmm1 |
| ; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm2 = [63,63] |
| ; AVX512F-NEXT: vpand %xmm2, %xmm1, %xmm3 |
| ; AVX512F-NEXT: vpsllq %xmm3, %zmm0, %zmm3 |
| ; AVX512F-NEXT: vextracti64x4 $1, %zmm3, %ymm3 |
| ; AVX512F-NEXT: valignq {{.*#+}} zmm0 = zmm0[7,0,1,2,3,4,5,6] |
| ; AVX512F-NEXT: vpsrlq $1, %zmm0, %zmm0 |
| ; AVX512F-NEXT: vpandn %xmm2, %xmm1, %xmm1 |
| ; AVX512F-NEXT: vpsrlq %xmm1, %zmm0, %zmm0 |
| ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0 |
| ; AVX512F-NEXT: vpor %ymm0, %ymm3, %ymm0 |
| ; AVX512F-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512F-NEXT: retq |
| ; |
| ; AVX512VL-LABEL: fshl_i256_vector: |
| ; AVX512VL: # %bb.0: |
| ; AVX512VL-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 |
| ; AVX512VL-NEXT: movq %rdi, %rax |
| ; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 |
| ; AVX512VL-NEXT: movzbl %sil, %ecx |
| ; AVX512VL-NEXT: vpbroadcastq %rcx, %xmm1 |
| ; AVX512VL-NEXT: # kill: def $ecx killed $ecx killed $rcx |
| ; AVX512VL-NEXT: shrl $6, %ecx |
| ; AVX512VL-NEXT: movl $-1, %edx |
| ; AVX512VL-NEXT: shlxl %ecx, %edx, %ecx |
| ; AVX512VL-NEXT: kmovd %ecx, %k1 |
| ; AVX512VL-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm2 = [63,63] |
| ; AVX512VL-NEXT: vpand %xmm2, %xmm1, %xmm3 |
| ; AVX512VL-NEXT: vpsllq %xmm3, %zmm0, %zmm3 |
| ; AVX512VL-NEXT: vextracti64x4 $1, %zmm3, %ymm3 |
| ; AVX512VL-NEXT: valignq {{.*#+}} zmm0 = zmm0[7,0,1,2,3,4,5,6] |
| ; AVX512VL-NEXT: vpsrlq $1, %zmm0, %zmm0 |
| ; AVX512VL-NEXT: vpandn %xmm2, %xmm1, %xmm1 |
| ; AVX512VL-NEXT: vpsrlq %xmm1, %zmm0, %zmm0 |
| ; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 |
| ; AVX512VL-NEXT: vpor %ymm0, %ymm3, %ymm0 |
| ; AVX512VL-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512VL-NEXT: vzeroupper |
| ; AVX512VL-NEXT: retq |
| ; |
| ; AVX512VBMI-LABEL: fshl_i256_vector: |
| ; AVX512VBMI: # %bb.0: |
| ; AVX512VBMI-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 |
| ; AVX512VBMI-NEXT: movq %rdi, %rax |
| ; AVX512VBMI-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 |
| ; AVX512VBMI-NEXT: movzbl %sil, %ecx |
| ; AVX512VBMI-NEXT: movl %ecx, %edx |
| ; AVX512VBMI-NEXT: shrl $6, %edx |
| ; AVX512VBMI-NEXT: movl $-1, %esi |
| ; AVX512VBMI-NEXT: shlxl %edx, %esi, %edx |
| ; AVX512VBMI-NEXT: kmovd %edx, %k1 |
| ; AVX512VBMI-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm1 |
| ; AVX512VBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2 |
| ; AVX512VBMI-NEXT: valignq {{.*#+}} zmm2 = zmm2[7],zmm0[0,1,2,3,4,5,6] |
| ; AVX512VBMI-NEXT: vpshldvq %zmm1, %zmm2, %zmm0 |
| ; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm0, (%rdi) |
| ; AVX512VBMI-NEXT: vzeroupper |
| ; AVX512VBMI-NEXT: retq |
| %a0 = bitcast <4 x i64> %v0 to i256 |
| %a1 = bitcast <4 x i64> %v1 to i256 |
| %r = call i256 @llvm.fshl.i256(i256 %a0, i256 %a1, i256 %a2) |
| ret i256 %r |
| } |
| |
| define i256 @fshr_i256_vector(<4 x i64> %v0, <4 x i64> %v1, i256 %a2) nounwind { |
| ; SSE2-LABEL: fshr_i256_vector: |
| ; SSE2: # %bb.0: |
| ; SSE2-NEXT: movq %rsi, %rcx |
| ; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3] |
| ; SSE2-NEXT: movq %xmm1, %rdx |
| ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] |
| ; SSE2-NEXT: movq %xmm1, %rax |
| ; SSE2-NEXT: movq %xmm0, %r10 |
| ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3] |
| ; SSE2-NEXT: movq %xmm0, %rsi |
| ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3] |
| ; SSE2-NEXT: movq %xmm0, %r8 |
| ; SSE2-NEXT: movq %xmm2, %r11 |
| ; SSE2-NEXT: movq %xmm3, %r9 |
| ; SSE2-NEXT: testb %cl, %cl |
| ; SSE2-NEXT: cmovsq %r9, %r11 |
| ; SSE2-NEXT: cmovsq %r8, %rsi |
| ; SSE2-NEXT: cmovsq %r10, %r9 |
| ; SSE2-NEXT: cmovsq %rax, %r8 |
| ; SSE2-NEXT: cmovnsq %r10, %rdx |
| ; SSE2-NEXT: movq %xmm4, %r10 |
| ; SSE2-NEXT: cmovnsq %rax, %r10 |
| ; SSE2-NEXT: testb $64, %cl |
| ; SSE2-NEXT: cmoveq %rdx, %r10 |
| ; SSE2-NEXT: cmoveq %r8, %rdx |
| ; SSE2-NEXT: cmoveq %r9, %r8 |
| ; SSE2-NEXT: cmoveq %rsi, %r9 |
| ; SSE2-NEXT: cmoveq %r11, %rsi |
| ; SSE2-NEXT: shrdq %cl, %r9, %rsi |
| ; SSE2-NEXT: shrdq %cl, %r8, %r9 |
| ; SSE2-NEXT: shrdq %cl, %rdx, %r8 |
| ; SSE2-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; SSE2-NEXT: shrdq %cl, %r10, %rdx |
| ; SSE2-NEXT: movq %rdi, %rax |
| ; SSE2-NEXT: movq %rdx, 24(%rdi) |
| ; SSE2-NEXT: movq %r8, 16(%rdi) |
| ; SSE2-NEXT: movq %r9, 8(%rdi) |
| ; SSE2-NEXT: movq %rsi, (%rdi) |
| ; SSE2-NEXT: retq |
| ; |
| ; SSE42-LABEL: fshr_i256_vector: |
| ; SSE42: # %bb.0: |
| ; SSE42-NEXT: pushq %rbx |
| ; SSE42-NEXT: movq %rsi, %rcx |
| ; SSE42-NEXT: pextrq $1, %xmm1, %rax |
| ; SSE42-NEXT: movq %xmm1, %rdx |
| ; SSE42-NEXT: pextrq $1, %xmm0, %r10 |
| ; SSE42-NEXT: pextrq $1, %xmm2, %rsi |
| ; SSE42-NEXT: movq %xmm0, %r11 |
| ; SSE42-NEXT: pextrq $1, %xmm3, %r8 |
| ; SSE42-NEXT: movq %xmm2, %rbx |
| ; SSE42-NEXT: movq %xmm3, %r9 |
| ; SSE42-NEXT: testb %cl, %cl |
| ; SSE42-NEXT: cmovsq %r9, %rbx |
| ; SSE42-NEXT: cmovsq %r8, %rsi |
| ; SSE42-NEXT: cmovsq %r11, %r9 |
| ; SSE42-NEXT: cmovsq %r10, %r8 |
| ; SSE42-NEXT: cmovnsq %r11, %rdx |
| ; SSE42-NEXT: cmovnsq %r10, %rax |
| ; SSE42-NEXT: testb $64, %cl |
| ; SSE42-NEXT: cmoveq %rdx, %rax |
| ; SSE42-NEXT: cmoveq %r8, %rdx |
| ; SSE42-NEXT: cmoveq %r9, %r8 |
| ; SSE42-NEXT: cmoveq %rsi, %r9 |
| ; SSE42-NEXT: cmoveq %rbx, %rsi |
| ; SSE42-NEXT: shrdq %cl, %r9, %rsi |
| ; SSE42-NEXT: shrdq %cl, %r8, %r9 |
| ; SSE42-NEXT: shrdq %cl, %rdx, %r8 |
| ; SSE42-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; SSE42-NEXT: shrdq %cl, %rax, %rdx |
| ; SSE42-NEXT: movq %rdi, %rax |
| ; SSE42-NEXT: movq %rdx, 24(%rdi) |
| ; SSE42-NEXT: movq %r8, 16(%rdi) |
| ; SSE42-NEXT: movq %r9, 8(%rdi) |
| ; SSE42-NEXT: movq %rsi, (%rdi) |
| ; SSE42-NEXT: popq %rbx |
| ; SSE42-NEXT: retq |
| ; |
| ; AVX2-LABEL: fshr_i256_vector: |
| ; AVX2: # %bb.0: |
| ; AVX2-NEXT: pushq %rbx |
| ; AVX2-NEXT: movq %rsi, %rcx |
| ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 |
| ; AVX2-NEXT: vpextrq $1, %xmm2, %rdx |
| ; AVX2-NEXT: vpextrq $1, %xmm0, %rax |
| ; AVX2-NEXT: vpextrq $1, %xmm1, %rsi |
| ; AVX2-NEXT: vmovq %xmm2, %r8 |
| ; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 |
| ; AVX2-NEXT: vpextrq $1, %xmm2, %r9 |
| ; AVX2-NEXT: vmovq %xmm0, %r11 |
| ; AVX2-NEXT: vmovq %xmm1, %rbx |
| ; AVX2-NEXT: vmovq %xmm2, %r10 |
| ; AVX2-NEXT: testb %cl, %cl |
| ; AVX2-NEXT: cmovsq %r10, %rbx |
| ; AVX2-NEXT: cmovsq %r9, %rsi |
| ; AVX2-NEXT: cmovsq %r11, %r10 |
| ; AVX2-NEXT: cmovsq %rax, %r9 |
| ; AVX2-NEXT: cmovnsq %r11, %r8 |
| ; AVX2-NEXT: cmovnsq %rax, %rdx |
| ; AVX2-NEXT: movq %rdi, %rax |
| ; AVX2-NEXT: testb $64, %cl |
| ; AVX2-NEXT: cmoveq %r8, %rdx |
| ; AVX2-NEXT: cmoveq %r9, %r8 |
| ; AVX2-NEXT: cmoveq %r10, %r9 |
| ; AVX2-NEXT: cmoveq %rsi, %r10 |
| ; AVX2-NEXT: cmoveq %rbx, %rsi |
| ; AVX2-NEXT: shrdq %cl, %r10, %rsi |
| ; AVX2-NEXT: shrdq %cl, %r9, %r10 |
| ; AVX2-NEXT: shrdq %cl, %r8, %r9 |
| ; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; AVX2-NEXT: shrdq %cl, %rdx, %r8 |
| ; AVX2-NEXT: movq %r8, 24(%rdi) |
| ; AVX2-NEXT: movq %r9, 16(%rdi) |
| ; AVX2-NEXT: movq %r10, 8(%rdi) |
| ; AVX2-NEXT: movq %rsi, (%rdi) |
| ; AVX2-NEXT: popq %rbx |
| ; AVX2-NEXT: vzeroupper |
| ; AVX2-NEXT: retq |
| ; |
| ; AVX512F-LABEL: fshr_i256_vector: |
| ; AVX512F: # %bb.0: |
| ; AVX512F-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 |
| ; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 |
| ; AVX512F-NEXT: movzbl %sil, %eax |
| ; AVX512F-NEXT: vmovq %rax, %xmm1 |
| ; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax |
| ; AVX512F-NEXT: shrl $6, %eax |
| ; AVX512F-NEXT: movl $-1, %ecx |
| ; AVX512F-NEXT: shlxl %eax, %ecx, %eax |
| ; AVX512F-NEXT: kmovw %eax, %k1 |
| ; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512F-NEXT: vpbroadcastq %xmm1, %xmm1 |
| ; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm2 = [63,63] |
| ; AVX512F-NEXT: vpand %xmm2, %xmm1, %xmm3 |
| ; AVX512F-NEXT: vpsrlq %xmm3, %ymm0, %ymm3 |
| ; AVX512F-NEXT: valignq {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7,0] |
| ; AVX512F-NEXT: vpaddq %ymm0, %ymm0, %ymm0 |
| ; AVX512F-NEXT: vpandn %xmm2, %xmm1, %xmm1 |
| ; AVX512F-NEXT: vpsllq %xmm1, %ymm0, %ymm0 |
| ; AVX512F-NEXT: movq %rdi, %rax |
| ; AVX512F-NEXT: vpor %ymm3, %ymm0, %ymm0 |
| ; AVX512F-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512F-NEXT: retq |
| ; |
| ; AVX512VL-LABEL: fshr_i256_vector: |
| ; AVX512VL: # %bb.0: |
| ; AVX512VL-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 |
| ; AVX512VL-NEXT: movq %rdi, %rax |
| ; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 |
| ; AVX512VL-NEXT: movzbl %sil, %ecx |
| ; AVX512VL-NEXT: movl %ecx, %edx |
| ; AVX512VL-NEXT: shrl $6, %edx |
| ; AVX512VL-NEXT: movl $-1, %esi |
| ; AVX512VL-NEXT: shlxl %edx, %esi, %edx |
| ; AVX512VL-NEXT: kmovd %edx, %k1 |
| ; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm1 = [63,63] |
| ; AVX512VL-NEXT: vpbroadcastq %rcx, %xmm2 |
| ; AVX512VL-NEXT: vpand %xmm1, %xmm2, %xmm3 |
| ; AVX512VL-NEXT: vpsrlq %xmm3, %ymm0, %ymm3 |
| ; AVX512VL-NEXT: valignq {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7,0] |
| ; AVX512VL-NEXT: vpaddq %ymm0, %ymm0, %ymm0 |
| ; AVX512VL-NEXT: vpandn %xmm1, %xmm2, %xmm1 |
| ; AVX512VL-NEXT: vpsllq %xmm1, %ymm0, %ymm0 |
| ; AVX512VL-NEXT: vpor %ymm3, %ymm0, %ymm0 |
| ; AVX512VL-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512VL-NEXT: vzeroupper |
| ; AVX512VL-NEXT: retq |
| ; |
| ; AVX512VBMI-LABEL: fshr_i256_vector: |
| ; AVX512VBMI: # %bb.0: |
| ; AVX512VBMI-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1 |
| ; AVX512VBMI-NEXT: movq %rdi, %rax |
| ; AVX512VBMI-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 |
| ; AVX512VBMI-NEXT: movzbl %sil, %ecx |
| ; AVX512VBMI-NEXT: movl %ecx, %edx |
| ; AVX512VBMI-NEXT: shrl $6, %edx |
| ; AVX512VBMI-NEXT: movl $-1, %esi |
| ; AVX512VBMI-NEXT: shlxl %edx, %esi, %edx |
| ; AVX512VBMI-NEXT: kmovd %edx, %k1 |
| ; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm1 |
| ; AVX512VBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2 |
| ; AVX512VBMI-NEXT: valignq {{.*#+}} zmm2 = zmm0[1,2,3,4,5,6,7],zmm2[0] |
| ; AVX512VBMI-NEXT: vpshrdvq %zmm1, %zmm2, %zmm0 |
| ; AVX512VBMI-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512VBMI-NEXT: vzeroupper |
| ; AVX512VBMI-NEXT: retq |
| %a0 = bitcast <4 x i64> %v0 to i256 |
| %a1 = bitcast <4 x i64> %v1 to i256 |
| %r = call i256 @llvm.fshr.i256(i256 %a0, i256 %a1, i256 %a2) |
| ret i256 %r |
| } |
| |
| define i256 @fshl_rot_i256_vector(<4 x i64> %v0, i256 %a2) nounwind { |
| ; SSE2-LABEL: fshl_rot_i256_vector: |
| ; SSE2: # %bb.0: |
| ; SSE2-NEXT: movq %rsi, %rcx |
| ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3] |
| ; SSE2-NEXT: movq %xmm2, %rax |
| ; SSE2-NEXT: movq %xmm1, %rdx |
| ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] |
| ; SSE2-NEXT: movq %xmm1, %rsi |
| ; SSE2-NEXT: testb $-128, %cl |
| ; SSE2-NEXT: movq %rsi, %r8 |
| ; SSE2-NEXT: cmovneq %rax, %r8 |
| ; SSE2-NEXT: movq %xmm0, %r9 |
| ; SSE2-NEXT: movq %r9, %r10 |
| ; SSE2-NEXT: cmovneq %rdx, %r10 |
| ; SSE2-NEXT: cmovneq %rsi, %rax |
| ; SSE2-NEXT: cmovneq %r9, %rdx |
| ; SSE2-NEXT: testb $64, %cl |
| ; SSE2-NEXT: movq %rdx, %rsi |
| ; SSE2-NEXT: cmovneq %rax, %rsi |
| ; SSE2-NEXT: cmovneq %r10, %rax |
| ; SSE2-NEXT: cmoveq %r8, %rdx |
| ; SSE2-NEXT: cmovneq %r8, %r10 |
| ; SSE2-NEXT: movq %r10, %r8 |
| ; SSE2-NEXT: shldq %cl, %rdx, %r8 |
| ; SSE2-NEXT: movq %rax, %r9 |
| ; SSE2-NEXT: shldq %cl, %r10, %r9 |
| ; SSE2-NEXT: movq %rsi, %r10 |
| ; SSE2-NEXT: shldq %cl, %rax, %r10 |
| ; SSE2-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; SSE2-NEXT: shldq %cl, %rsi, %rdx |
| ; SSE2-NEXT: movq %rdi, %rax |
| ; SSE2-NEXT: movq %rdx, 24(%rdi) |
| ; SSE2-NEXT: movq %r10, 16(%rdi) |
| ; SSE2-NEXT: movq %r9, 8(%rdi) |
| ; SSE2-NEXT: movq %r8, (%rdi) |
| ; SSE2-NEXT: retq |
| ; |
| ; SSE42-LABEL: fshl_rot_i256_vector: |
| ; SSE42: # %bb.0: |
| ; SSE42-NEXT: movq %rsi, %rcx |
| ; SSE42-NEXT: movq %xmm1, %rdx |
| ; SSE42-NEXT: pextrq $1, %xmm0, %rax |
| ; SSE42-NEXT: pextrq $1, %xmm1, %rsi |
| ; SSE42-NEXT: testb $-128, %cl |
| ; SSE42-NEXT: movq %rsi, %r8 |
| ; SSE42-NEXT: cmovneq %rax, %r8 |
| ; SSE42-NEXT: movq %xmm0, %r9 |
| ; SSE42-NEXT: movq %r9, %r10 |
| ; SSE42-NEXT: cmovneq %rdx, %r10 |
| ; SSE42-NEXT: cmovneq %rsi, %rax |
| ; SSE42-NEXT: cmovneq %r9, %rdx |
| ; SSE42-NEXT: testb $64, %cl |
| ; SSE42-NEXT: movq %rdx, %rsi |
| ; SSE42-NEXT: cmovneq %rax, %rsi |
| ; SSE42-NEXT: cmovneq %r10, %rax |
| ; SSE42-NEXT: cmoveq %r8, %rdx |
| ; SSE42-NEXT: cmovneq %r8, %r10 |
| ; SSE42-NEXT: movq %r10, %r8 |
| ; SSE42-NEXT: shldq %cl, %rdx, %r8 |
| ; SSE42-NEXT: movq %rax, %r9 |
| ; SSE42-NEXT: shldq %cl, %r10, %r9 |
| ; SSE42-NEXT: movq %rsi, %r10 |
| ; SSE42-NEXT: shldq %cl, %rax, %r10 |
| ; SSE42-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; SSE42-NEXT: shldq %cl, %rsi, %rdx |
| ; SSE42-NEXT: movq %rdi, %rax |
| ; SSE42-NEXT: movq %rdx, 24(%rdi) |
| ; SSE42-NEXT: movq %r10, 16(%rdi) |
| ; SSE42-NEXT: movq %r9, 8(%rdi) |
| ; SSE42-NEXT: movq %r8, (%rdi) |
| ; SSE42-NEXT: retq |
| ; |
| ; AVX2-LABEL: fshl_rot_i256_vector: |
| ; AVX2: # %bb.0: |
| ; AVX2-NEXT: movq %rsi, %rcx |
| ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 |
| ; AVX2-NEXT: vmovq %xmm1, %rdx |
| ; AVX2-NEXT: vpextrq $1, %xmm0, %rsi |
| ; AVX2-NEXT: vpextrq $1, %xmm1, %rax |
| ; AVX2-NEXT: vmovq %xmm0, %r8 |
| ; AVX2-NEXT: testb $-128, %cl |
| ; AVX2-NEXT: movq %rax, %r9 |
| ; AVX2-NEXT: cmovneq %rsi, %r9 |
| ; AVX2-NEXT: movq %r8, %r10 |
| ; AVX2-NEXT: cmovneq %rdx, %r10 |
| ; AVX2-NEXT: cmovneq %rax, %rsi |
| ; AVX2-NEXT: movq %rdi, %rax |
| ; AVX2-NEXT: cmovneq %r8, %rdx |
| ; AVX2-NEXT: testb $64, %cl |
| ; AVX2-NEXT: movq %rdx, %rdi |
| ; AVX2-NEXT: cmovneq %rsi, %rdi |
| ; AVX2-NEXT: cmovneq %r10, %rsi |
| ; AVX2-NEXT: cmoveq %r9, %rdx |
| ; AVX2-NEXT: cmovneq %r9, %r10 |
| ; AVX2-NEXT: movq %r10, %r8 |
| ; AVX2-NEXT: shldq %cl, %rdx, %r8 |
| ; AVX2-NEXT: movq %rsi, %r9 |
| ; AVX2-NEXT: shldq %cl, %r10, %r9 |
| ; AVX2-NEXT: movq %rdi, %r10 |
| ; AVX2-NEXT: shldq %cl, %rsi, %r10 |
| ; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; AVX2-NEXT: shldq %cl, %rdi, %rdx |
| ; AVX2-NEXT: movq %rdx, 24(%rax) |
| ; AVX2-NEXT: movq %r10, 16(%rax) |
| ; AVX2-NEXT: movq %r9, 8(%rax) |
| ; AVX2-NEXT: movq %r8, (%rax) |
| ; AVX2-NEXT: vzeroupper |
| ; AVX2-NEXT: retq |
| ; |
| ; AVX512F-LABEL: fshl_rot_i256_vector: |
| ; AVX512F: # %bb.0: |
| ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 |
| ; AVX512F-NEXT: movq %rdi, %rax |
| ; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 |
| ; AVX512F-NEXT: movzbl %sil, %ecx |
| ; AVX512F-NEXT: vmovq %rcx, %xmm1 |
| ; AVX512F-NEXT: # kill: def $ecx killed $ecx killed $rcx |
| ; AVX512F-NEXT: shrl $6, %ecx |
| ; AVX512F-NEXT: movl $-1, %edx |
| ; AVX512F-NEXT: shlxl %ecx, %edx, %ecx |
| ; AVX512F-NEXT: kmovw %ecx, %k1 |
| ; AVX512F-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512F-NEXT: valignq {{.*#+}} zmm2 = zmm0[7,0,1,2,3,4,5,6] |
| ; AVX512F-NEXT: vpsrlq $1, %zmm2, %zmm2 |
| ; AVX512F-NEXT: vpbroadcastq %xmm1, %xmm1 |
| ; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm3 = [63,63] |
| ; AVX512F-NEXT: vpandn %xmm3, %xmm1, %xmm4 |
| ; AVX512F-NEXT: vpsrlq %xmm4, %zmm2, %zmm2 |
| ; AVX512F-NEXT: vpand %xmm3, %xmm1, %xmm1 |
| ; AVX512F-NEXT: vpsllq %xmm1, %zmm0, %zmm0 |
| ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0 |
| ; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm1 |
| ; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0 |
| ; AVX512F-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512F-NEXT: retq |
| ; |
| ; AVX512VL-LABEL: fshl_rot_i256_vector: |
| ; AVX512VL: # %bb.0: |
| ; AVX512VL-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 |
| ; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 |
| ; AVX512VL-NEXT: movzbl %sil, %eax |
| ; AVX512VL-NEXT: vpbroadcastq %rax, %xmm1 |
| ; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax |
| ; AVX512VL-NEXT: shrl $6, %eax |
| ; AVX512VL-NEXT: movl $-1, %ecx |
| ; AVX512VL-NEXT: shlxl %eax, %ecx, %eax |
| ; AVX512VL-NEXT: kmovd %eax, %k1 |
| ; AVX512VL-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VL-NEXT: valignq {{.*#+}} zmm2 = zmm0[7,0,1,2,3,4,5,6] |
| ; AVX512VL-NEXT: vpsrlq $1, %zmm2, %zmm2 |
| ; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm3 = [63,63] |
| ; AVX512VL-NEXT: vpandn %xmm3, %xmm1, %xmm4 |
| ; AVX512VL-NEXT: vpsrlq %xmm4, %zmm2, %zmm2 |
| ; AVX512VL-NEXT: vpand %xmm3, %xmm1, %xmm1 |
| ; AVX512VL-NEXT: vpsllq %xmm1, %zmm0, %zmm0 |
| ; AVX512VL-NEXT: movq %rdi, %rax |
| ; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm0 |
| ; AVX512VL-NEXT: vextracti64x4 $1, %zmm2, %ymm1 |
| ; AVX512VL-NEXT: vpor %ymm1, %ymm0, %ymm0 |
| ; AVX512VL-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512VL-NEXT: vzeroupper |
| ; AVX512VL-NEXT: retq |
| ; |
| ; AVX512VBMI-LABEL: fshl_rot_i256_vector: |
| ; AVX512VBMI: # %bb.0: |
| ; AVX512VBMI-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 |
| ; AVX512VBMI-NEXT: movq %rdi, %rax |
| ; AVX512VBMI-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 |
| ; AVX512VBMI-NEXT: movzbl %sil, %ecx |
| ; AVX512VBMI-NEXT: movl %ecx, %edx |
| ; AVX512VBMI-NEXT: shrl $6, %edx |
| ; AVX512VBMI-NEXT: movl $-1, %esi |
| ; AVX512VBMI-NEXT: shlxl %edx, %esi, %edx |
| ; AVX512VBMI-NEXT: kmovd %edx, %k1 |
| ; AVX512VBMI-NEXT: vpexpandq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm1 |
| ; AVX512VBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2 |
| ; AVX512VBMI-NEXT: valignq {{.*#+}} zmm2 = zmm2[7],zmm0[0,1,2,3,4,5,6] |
| ; AVX512VBMI-NEXT: vpshldvq %zmm1, %zmm2, %zmm0 |
| ; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm0, (%rdi) |
| ; AVX512VBMI-NEXT: vzeroupper |
| ; AVX512VBMI-NEXT: retq |
| %a0 = bitcast <4 x i64> %v0 to i256 |
| %r = call i256 @llvm.fshl.i256(i256 %a0, i256 %a0, i256 %a2) |
| ret i256 %r |
| } |
| |
| define i256 @fshr_rot_i256_vector(<4 x i64> %v0, i256 %a2) nounwind { |
| ; SSE2-LABEL: fshr_rot_i256_vector: |
| ; SSE2: # %bb.0: |
| ; SSE2-NEXT: movq %rsi, %rcx |
| ; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3] |
| ; SSE2-NEXT: movq %xmm2, %rdx |
| ; SSE2-NEXT: movq %xmm1, %rax |
| ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] |
| ; SSE2-NEXT: movq %xmm1, %r8 |
| ; SSE2-NEXT: testb %cl, %cl |
| ; SSE2-NEXT: movq %r8, %r9 |
| ; SSE2-NEXT: cmovnsq %rdx, %r9 |
| ; SSE2-NEXT: movq %xmm0, %r10 |
| ; SSE2-NEXT: movq %r10, %rsi |
| ; SSE2-NEXT: cmovnsq %rax, %rsi |
| ; SSE2-NEXT: cmovnsq %r8, %rdx |
| ; SSE2-NEXT: cmovnsq %r10, %rax |
| ; SSE2-NEXT: testb $64, %cl |
| ; SSE2-NEXT: movq %rax, %r8 |
| ; SSE2-NEXT: cmoveq %rdx, %r8 |
| ; SSE2-NEXT: cmoveq %rsi, %rdx |
| ; SSE2-NEXT: cmoveq %r9, %rsi |
| ; SSE2-NEXT: cmovneq %r9, %rax |
| ; SSE2-NEXT: movq %rax, %r9 |
| ; SSE2-NEXT: shrdq %cl, %rsi, %r9 |
| ; SSE2-NEXT: shrdq %cl, %rdx, %rsi |
| ; SSE2-NEXT: shrdq %cl, %r8, %rdx |
| ; SSE2-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; SSE2-NEXT: shrdq %cl, %rax, %r8 |
| ; SSE2-NEXT: movq %rdi, %rax |
| ; SSE2-NEXT: movq %r8, 24(%rdi) |
| ; SSE2-NEXT: movq %rdx, 16(%rdi) |
| ; SSE2-NEXT: movq %rsi, 8(%rdi) |
| ; SSE2-NEXT: movq %r9, (%rdi) |
| ; SSE2-NEXT: retq |
| ; |
| ; SSE42-LABEL: fshr_rot_i256_vector: |
| ; SSE42: # %bb.0: |
| ; SSE42-NEXT: movq %rsi, %rcx |
| ; SSE42-NEXT: movq %xmm1, %rax |
| ; SSE42-NEXT: pextrq $1, %xmm0, %rdx |
| ; SSE42-NEXT: pextrq $1, %xmm1, %r8 |
| ; SSE42-NEXT: testb %cl, %cl |
| ; SSE42-NEXT: movq %r8, %r9 |
| ; SSE42-NEXT: cmovnsq %rdx, %r9 |
| ; SSE42-NEXT: movq %xmm0, %r10 |
| ; SSE42-NEXT: movq %r10, %rsi |
| ; SSE42-NEXT: cmovnsq %rax, %rsi |
| ; SSE42-NEXT: cmovnsq %r8, %rdx |
| ; SSE42-NEXT: cmovnsq %r10, %rax |
| ; SSE42-NEXT: testb $64, %cl |
| ; SSE42-NEXT: movq %rax, %r8 |
| ; SSE42-NEXT: cmoveq %rdx, %r8 |
| ; SSE42-NEXT: cmoveq %rsi, %rdx |
| ; SSE42-NEXT: cmoveq %r9, %rsi |
| ; SSE42-NEXT: cmovneq %r9, %rax |
| ; SSE42-NEXT: movq %rax, %r9 |
| ; SSE42-NEXT: shrdq %cl, %rsi, %r9 |
| ; SSE42-NEXT: shrdq %cl, %rdx, %rsi |
| ; SSE42-NEXT: shrdq %cl, %r8, %rdx |
| ; SSE42-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; SSE42-NEXT: shrdq %cl, %rax, %r8 |
| ; SSE42-NEXT: movq %rdi, %rax |
| ; SSE42-NEXT: movq %r8, 24(%rdi) |
| ; SSE42-NEXT: movq %rdx, 16(%rdi) |
| ; SSE42-NEXT: movq %rsi, 8(%rdi) |
| ; SSE42-NEXT: movq %r9, (%rdi) |
| ; SSE42-NEXT: retq |
| ; |
| ; AVX2-LABEL: fshr_rot_i256_vector: |
| ; AVX2: # %bb.0: |
| ; AVX2-NEXT: movq %rsi, %rcx |
| ; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 |
| ; AVX2-NEXT: vmovq %xmm1, %rdx |
| ; AVX2-NEXT: vpextrq $1, %xmm0, %rsi |
| ; AVX2-NEXT: vpextrq $1, %xmm1, %rax |
| ; AVX2-NEXT: vmovq %xmm0, %r9 |
| ; AVX2-NEXT: testb %cl, %cl |
| ; AVX2-NEXT: movq %rax, %r10 |
| ; AVX2-NEXT: cmovnsq %rsi, %r10 |
| ; AVX2-NEXT: movq %r9, %r8 |
| ; AVX2-NEXT: cmovnsq %rdx, %r8 |
| ; AVX2-NEXT: cmovnsq %rax, %rsi |
| ; AVX2-NEXT: movq %rdi, %rax |
| ; AVX2-NEXT: cmovnsq %r9, %rdx |
| ; AVX2-NEXT: testb $64, %cl |
| ; AVX2-NEXT: movq %rdx, %rdi |
| ; AVX2-NEXT: cmoveq %rsi, %rdi |
| ; AVX2-NEXT: cmoveq %r8, %rsi |
| ; AVX2-NEXT: cmoveq %r10, %r8 |
| ; AVX2-NEXT: cmovneq %r10, %rdx |
| ; AVX2-NEXT: movq %rdx, %r9 |
| ; AVX2-NEXT: shrdq %cl, %r8, %r9 |
| ; AVX2-NEXT: shrdq %cl, %rsi, %r8 |
| ; AVX2-NEXT: shrdq %cl, %rdi, %rsi |
| ; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx |
| ; AVX2-NEXT: shrdq %cl, %rdx, %rdi |
| ; AVX2-NEXT: movq %rdi, 24(%rax) |
| ; AVX2-NEXT: movq %rsi, 16(%rax) |
| ; AVX2-NEXT: movq %r8, 8(%rax) |
| ; AVX2-NEXT: movq %r9, (%rax) |
| ; AVX2-NEXT: vzeroupper |
| ; AVX2-NEXT: retq |
| ; |
| ; AVX512F-LABEL: fshr_rot_i256_vector: |
| ; AVX512F: # %bb.0: |
| ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 |
| ; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 |
| ; AVX512F-NEXT: movzbl %sil, %eax |
| ; AVX512F-NEXT: vmovq %rax, %xmm1 |
| ; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax |
| ; AVX512F-NEXT: shrl $6, %eax |
| ; AVX512F-NEXT: movl $-1, %ecx |
| ; AVX512F-NEXT: shlxl %eax, %ecx, %eax |
| ; AVX512F-NEXT: kmovw %eax, %k1 |
| ; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512F-NEXT: vpbroadcastq %xmm1, %xmm1 |
| ; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm2 = [63,63] |
| ; AVX512F-NEXT: vpand %xmm2, %xmm1, %xmm3 |
| ; AVX512F-NEXT: vpsrlq %xmm3, %ymm0, %ymm3 |
| ; AVX512F-NEXT: valignq {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7,0] |
| ; AVX512F-NEXT: vpaddq %ymm0, %ymm0, %ymm0 |
| ; AVX512F-NEXT: vpandn %xmm2, %xmm1, %xmm1 |
| ; AVX512F-NEXT: vpsllq %xmm1, %ymm0, %ymm0 |
| ; AVX512F-NEXT: movq %rdi, %rax |
| ; AVX512F-NEXT: vpor %ymm3, %ymm0, %ymm0 |
| ; AVX512F-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512F-NEXT: retq |
| ; |
| ; AVX512VL-LABEL: fshr_rot_i256_vector: |
| ; AVX512VL: # %bb.0: |
| ; AVX512VL-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 |
| ; AVX512VL-NEXT: movq %rdi, %rax |
| ; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 |
| ; AVX512VL-NEXT: movzbl %sil, %ecx |
| ; AVX512VL-NEXT: movl %ecx, %edx |
| ; AVX512VL-NEXT: shrl $6, %edx |
| ; AVX512VL-NEXT: movl $-1, %esi |
| ; AVX512VL-NEXT: shlxl %edx, %esi, %edx |
| ; AVX512VL-NEXT: kmovd %edx, %k1 |
| ; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm1 = [63,63] |
| ; AVX512VL-NEXT: vpbroadcastq %rcx, %xmm2 |
| ; AVX512VL-NEXT: vpand %xmm1, %xmm2, %xmm3 |
| ; AVX512VL-NEXT: vpsrlq %xmm3, %ymm0, %ymm3 |
| ; AVX512VL-NEXT: valignq {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7,0] |
| ; AVX512VL-NEXT: vpaddq %ymm0, %ymm0, %ymm0 |
| ; AVX512VL-NEXT: vpandn %xmm1, %xmm2, %xmm1 |
| ; AVX512VL-NEXT: vpsllq %xmm1, %ymm0, %ymm0 |
| ; AVX512VL-NEXT: vpor %ymm3, %ymm0, %ymm0 |
| ; AVX512VL-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512VL-NEXT: vzeroupper |
| ; AVX512VL-NEXT: retq |
| ; |
| ; AVX512VBMI-LABEL: fshr_rot_i256_vector: |
| ; AVX512VBMI: # %bb.0: |
| ; AVX512VBMI-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 |
| ; AVX512VBMI-NEXT: movq %rdi, %rax |
| ; AVX512VBMI-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 |
| ; AVX512VBMI-NEXT: movzbl %sil, %ecx |
| ; AVX512VBMI-NEXT: movl %ecx, %edx |
| ; AVX512VBMI-NEXT: shrl $6, %edx |
| ; AVX512VBMI-NEXT: movl $-1, %esi |
| ; AVX512VBMI-NEXT: shlxl %edx, %esi, %edx |
| ; AVX512VBMI-NEXT: kmovd %edx, %k1 |
| ; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} |
| ; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm1 |
| ; AVX512VBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2 |
| ; AVX512VBMI-NEXT: valignq {{.*#+}} zmm2 = zmm0[1,2,3,4,5,6,7],zmm2[0] |
| ; AVX512VBMI-NEXT: vpshrdvq %zmm1, %zmm2, %zmm0 |
| ; AVX512VBMI-NEXT: vmovdqu %ymm0, (%rdi) |
| ; AVX512VBMI-NEXT: vzeroupper |
| ; AVX512VBMI-NEXT: retq |
| %a0 = bitcast <4 x i64> %v0 to i256 |
| %r = call i256 @llvm.fshr.i256(i256 %a0, i256 %a0, i256 %a2) |
| ret i256 %r |
| } |
| |
| ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: |
| ; AVX512: {{.*}} |
