Google Git
Sign in
llvm / llvm-project / llvm / e7e1a7ad6f753bcb646a0a80f8ac71809ba0829d / . / test / CodeGen / X86 / min-legal-vector-width.ll
blob: 8dffb2c8559264ee2efdc89d35057b6ad97284f7 [file] [log] [blame]
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -mattr=prefer-256-bit | FileCheck %s --check-prefixes=CHECK,CHECK-SKX
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -mattr=prefer-256-bit,avx512vbmi | FileCheck %s --check-prefixes=CHECK,CHECK-SKX,CHECK-SKX-VBMI
; Make sure CPUs default to prefer-256-bit. avx512vnni isn't interesting as it just adds an isel peephole for vpmaddwd+vpaddd
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cascadelake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cooperlake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=cannonlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-client | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-server | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
; This file primarily contains tests for specific places in X86ISelLowering.cpp that needed be made aware of the legalizer not allowing 512-bit vectors due to prefer-256-bit even though AVX512 is enabled.
define dso_local void @add256(<16 x i32>* %a, <16 x i32>* %b, <16 x i32>* %c) "min-legal-vector-width"="256" {
; CHECK-LABEL: add256:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rdi), %ymm0
; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT: vpaddd 32(%rsi), %ymm1, %ymm1
; CHECK-NEXT: vpaddd (%rsi), %ymm0, %ymm0
; CHECK-NEXT: vmovdqa %ymm0, (%rdx)
; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%d = load <16 x i32>, <16 x i32>* %a
%e = load <16 x i32>, <16 x i32>* %b
%f = add <16 x i32> %d, %e
store <16 x i32> %f, <16 x i32>* %c
ret void
}
define dso_local void @add512(<16 x i32>* %a, <16 x i32>* %b, <16 x i32>* %c) "min-legal-vector-width"="512" {
; CHECK-LABEL: add512:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa64 (%rdi), %zmm0
; CHECK-NEXT: vpaddd (%rsi), %zmm0, %zmm0
; CHECK-NEXT: vmovdqa64 %zmm0, (%rdx)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%d = load <16 x i32>, <16 x i32>* %a
%e = load <16 x i32>, <16 x i32>* %b
%f = add <16 x i32> %d, %e
store <16 x i32> %f, <16 x i32>* %c
ret void
}
define dso_local void @avg_v64i8_256(<64 x i8>* %a, <64 x i8>* %b) "min-legal-vector-width"="256" {
; CHECK-LABEL: avg_v64i8_256:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rsi), %ymm0
; CHECK-NEXT: vmovdqa 32(%rsi), %ymm1
; CHECK-NEXT: vpavgb (%rdi), %ymm0, %ymm0
; CHECK-NEXT: vpavgb 32(%rdi), %ymm1, %ymm1
; CHECK-NEXT: vmovdqu %ymm1, (%rax)
; CHECK-NEXT: vmovdqu %ymm0, (%rax)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%1 = load <64 x i8>, <64 x i8>* %a
%2 = load <64 x i8>, <64 x i8>* %b
%3 = zext <64 x i8> %1 to <64 x i32>
%4 = zext <64 x i8> %2 to <64 x i32>
%5 = add nuw nsw <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
%6 = add nuw nsw <64 x i32> %5, %4
%7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
%8 = trunc <64 x i32> %7 to <64 x i8>
store <64 x i8> %8, <64 x i8>* undef, align 4
ret void
}
define dso_local void @avg_v64i8_512(<64 x i8>* %a, <64 x i8>* %b) "min-legal-vector-width"="512" {
; CHECK-LABEL: avg_v64i8_512:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa64 (%rdi), %zmm0
; CHECK-NEXT: vpavgb (%rsi), %zmm0, %zmm0
; CHECK-NEXT: vmovdqu64 %zmm0, (%rax)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%1 = load <64 x i8>, <64 x i8>* %a
%2 = load <64 x i8>, <64 x i8>* %b
%3 = zext <64 x i8> %1 to <64 x i32>
%4 = zext <64 x i8> %2 to <64 x i32>
%5 = add nuw nsw <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
%6 = add nuw nsw <64 x i32> %5, %4
%7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
%8 = trunc <64 x i32> %7 to <64 x i8>
store <64 x i8> %8, <64 x i8>* undef, align 4
ret void
}
define dso_local void @pmaddwd_32_256(<32 x i16>* %APtr, <32 x i16>* %BPtr, <16 x i32>* %CPtr) "min-legal-vector-width"="256" {
; CHECK-LABEL: pmaddwd_32_256:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rdi), %ymm0
; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm1
; CHECK-NEXT: vpmaddwd (%rsi), %ymm0, %ymm0
; CHECK-NEXT: vmovdqa %ymm0, (%rdx)
; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%A = load <32 x i16>, <32 x i16>* %APtr
%B = load <32 x i16>, <32 x i16>* %BPtr
%a = sext <32 x i16> %A to <32 x i32>
%b = sext <32 x i16> %B to <32 x i32>
%m = mul nsw <32 x i32> %a, %b
%odd = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%even = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
%ret = add <16 x i32> %odd, %even
store <16 x i32> %ret, <16 x i32>* %CPtr
ret void
}
define dso_local void @pmaddwd_32_512(<32 x i16>* %APtr, <32 x i16>* %BPtr, <16 x i32>* %CPtr) "min-legal-vector-width"="512" {
; CHECK-LABEL: pmaddwd_32_512:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa64 (%rdi), %zmm0
; CHECK-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0
; CHECK-NEXT: vmovdqa64 %zmm0, (%rdx)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%A = load <32 x i16>, <32 x i16>* %APtr
%B = load <32 x i16>, <32 x i16>* %BPtr
%a = sext <32 x i16> %A to <32 x i32>
%b = sext <32 x i16> %B to <32 x i32>
%m = mul nsw <32 x i32> %a, %b
%odd = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
%even = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
%ret = add <16 x i32> %odd, %even
store <16 x i32> %ret, <16 x i32>* %CPtr
ret void
}
define dso_local void @psubus_64i8_max_256(<64 x i8>* %xptr, <64 x i8>* %yptr, <64 x i8>* %zptr) "min-legal-vector-width"="256" {
; CHECK-LABEL: psubus_64i8_max_256:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rdi), %ymm0
; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT: vpsubusb 32(%rsi), %ymm1, %ymm1
; CHECK-NEXT: vpsubusb (%rsi), %ymm0, %ymm0
; CHECK-NEXT: vmovdqa %ymm0, (%rdx)
; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%x = load <64 x i8>, <64 x i8>* %xptr
%y = load <64 x i8>, <64 x i8>* %yptr
%cmp = icmp ult <64 x i8> %x, %y
%max = select <64 x i1> %cmp, <64 x i8> %y, <64 x i8> %x
%res = sub <64 x i8> %max, %y
store <64 x i8> %res, <64 x i8>* %zptr
ret void
}
define dso_local void @psubus_64i8_max_512(<64 x i8>* %xptr, <64 x i8>* %yptr, <64 x i8>* %zptr) "min-legal-vector-width"="512" {
; CHECK-LABEL: psubus_64i8_max_512:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa64 (%rdi), %zmm0
; CHECK-NEXT: vpsubusb (%rsi), %zmm0, %zmm0
; CHECK-NEXT: vmovdqa64 %zmm0, (%rdx)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%x = load <64 x i8>, <64 x i8>* %xptr
%y = load <64 x i8>, <64 x i8>* %yptr
%cmp = icmp ult <64 x i8> %x, %y
%max = select <64 x i1> %cmp, <64 x i8> %y, <64 x i8> %x
%res = sub <64 x i8> %max, %y
store <64 x i8> %res, <64 x i8>* %zptr
ret void
}
define dso_local i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly, i32) "min-legal-vector-width"="256" {
; CHECK-SKX-LABEL: _Z9test_charPcS_i_256:
; CHECK-SKX: # %bb.0: # %entry
; CHECK-SKX-NEXT: movl %edx, %eax
; CHECK-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-SKX-NEXT: xorl %ecx, %ecx
; CHECK-SKX-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-SKX-NEXT: vpxor %xmm2, %xmm2, %xmm2
; CHECK-SKX-NEXT: .p2align 4, 0x90
; CHECK-SKX-NEXT: .LBB8_1: # %vector.body
; CHECK-SKX-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-SKX-NEXT: vpmovsxbw 16(%rdi,%rcx), %ymm3
; CHECK-SKX-NEXT: vpmovsxbw (%rdi,%rcx), %ymm4
; CHECK-SKX-NEXT: vpmovsxbw 16(%rsi,%rcx), %ymm5
; CHECK-SKX-NEXT: vpmaddwd %ymm3, %ymm5, %ymm3
; CHECK-SKX-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; CHECK-SKX-NEXT: vpmovsxbw (%rsi,%rcx), %ymm3
; CHECK-SKX-NEXT: vpmaddwd %ymm4, %ymm3, %ymm3
; CHECK-SKX-NEXT: vpaddd %ymm1, %ymm3, %ymm1
; CHECK-SKX-NEXT: addq $32, %rcx
; CHECK-SKX-NEXT: cmpq %rcx, %rax
; CHECK-SKX-NEXT: jne .LBB8_1
; CHECK-SKX-NEXT: # %bb.2: # %middle.block
; CHECK-SKX-NEXT: vpaddd %ymm0, %ymm1, %ymm1
; CHECK-SKX-NEXT: vpaddd %ymm0, %ymm2, %ymm0
; CHECK-SKX-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; CHECK-SKX-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vpsrlq $32, %xmm0, %xmm1
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vmovd %xmm0, %eax
; CHECK-SKX-NEXT: vzeroupper
; CHECK-SKX-NEXT: retq
;
; CHECK-AVX512-LABEL: _Z9test_charPcS_i_256:
; CHECK-AVX512: # %bb.0: # %entry
; CHECK-AVX512-NEXT: movl %edx, %eax
; CHECK-AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-AVX512-NEXT: xorl %ecx, %ecx
; CHECK-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
; CHECK-AVX512-NEXT: .p2align 4, 0x90
; CHECK-AVX512-NEXT: .LBB8_1: # %vector.body
; CHECK-AVX512-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-AVX512-NEXT: vpmovsxbw 16(%rdi,%rcx), %ymm3
; CHECK-AVX512-NEXT: vpmovsxbw (%rdi,%rcx), %ymm4
; CHECK-AVX512-NEXT: vpmovsxbw 16(%rsi,%rcx), %ymm5
; CHECK-AVX512-NEXT: vpmaddwd %ymm3, %ymm5, %ymm3
; CHECK-AVX512-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; CHECK-AVX512-NEXT: vpmovsxbw (%rsi,%rcx), %ymm3
; CHECK-AVX512-NEXT: vpmaddwd %ymm4, %ymm3, %ymm3
; CHECK-AVX512-NEXT: vpaddd %ymm1, %ymm3, %ymm1
; CHECK-AVX512-NEXT: addq $32, %rcx
; CHECK-AVX512-NEXT: cmpq %rcx, %rax
; CHECK-AVX512-NEXT: jne .LBB8_1
; CHECK-AVX512-NEXT: # %bb.2: # %middle.block
; CHECK-AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm1
; CHECK-AVX512-NEXT: vpaddd %ymm0, %ymm2, %ymm0
; CHECK-AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; CHECK-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vpsrlq $32, %xmm0, %xmm1
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vmovd %xmm0, %eax
; CHECK-AVX512-NEXT: vzeroupper
; CHECK-AVX512-NEXT: retq
;
; CHECK-VBMI-LABEL: _Z9test_charPcS_i_256:
; CHECK-VBMI: # %bb.0: # %entry
; CHECK-VBMI-NEXT: movl %edx, %eax
; CHECK-VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-VBMI-NEXT: xorl %ecx, %ecx
; CHECK-VBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-VBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2
; CHECK-VBMI-NEXT: .p2align 4, 0x90
; CHECK-VBMI-NEXT: .LBB8_1: # %vector.body
; CHECK-VBMI-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-VBMI-NEXT: vpmovsxbw 16(%rdi,%rcx), %ymm3
; CHECK-VBMI-NEXT: vpmovsxbw (%rdi,%rcx), %ymm4
; CHECK-VBMI-NEXT: vpmovsxbw 16(%rsi,%rcx), %ymm5
; CHECK-VBMI-NEXT: vpmaddwd %ymm3, %ymm5, %ymm3
; CHECK-VBMI-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; CHECK-VBMI-NEXT: vpmovsxbw (%rsi,%rcx), %ymm3
; CHECK-VBMI-NEXT: vpmaddwd %ymm4, %ymm3, %ymm3
; CHECK-VBMI-NEXT: vpaddd %ymm1, %ymm3, %ymm1
; CHECK-VBMI-NEXT: addq $32, %rcx
; CHECK-VBMI-NEXT: cmpq %rcx, %rax
; CHECK-VBMI-NEXT: jne .LBB8_1
; CHECK-VBMI-NEXT: # %bb.2: # %middle.block
; CHECK-VBMI-NEXT: vpaddd %ymm0, %ymm1, %ymm1
; CHECK-VBMI-NEXT: vpaddd %ymm0, %ymm2, %ymm0
; CHECK-VBMI-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; CHECK-VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vmovd %xmm0, %eax
; CHECK-VBMI-NEXT: vzeroupper
; CHECK-VBMI-NEXT: retq
entry:
%3 = zext i32 %2 to i64
br label %vector.body
vector.body:
%index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
%vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
%4 = getelementptr inbounds i8, i8* %0, i64 %index
%5 = bitcast i8* %4 to <32 x i8>*
%wide.load = load <32 x i8>, <32 x i8>* %5, align 1
%6 = sext <32 x i8> %wide.load to <32 x i32>
%7 = getelementptr inbounds i8, i8* %1, i64 %index
%8 = bitcast i8* %7 to <32 x i8>*
%wide.load14 = load <32 x i8>, <32 x i8>* %8, align 1
%9 = sext <32 x i8> %wide.load14 to <32 x i32>
%10 = mul nsw <32 x i32> %9, %6
%11 = add nsw <32 x i32> %10, %vec.phi
%index.next = add i64 %index, 32
%12 = icmp eq i64 %index.next, %3
br i1 %12, label %middle.block, label %vector.body
middle.block:
%rdx.shuf1 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx1 = add <32 x i32> %11, %rdx.shuf1
%rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf
%rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx32 = add <32 x i32> %bin.rdx, %rdx.shuf15
%rdx.shuf17 = shufflevector <32 x i32> %bin.rdx32, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx18 = add <32 x i32> %bin.rdx32, %rdx.shuf17
%rdx.shuf19 = shufflevector <32 x i32> %bin.rdx18, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx20 = add <32 x i32> %bin.rdx18, %rdx.shuf19
%13 = extractelement <32 x i32> %bin.rdx20, i32 0
ret i32 %13
}
define dso_local i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly, i32) "min-legal-vector-width"="512" {
; CHECK-SKX-LABEL: _Z9test_charPcS_i_512:
; CHECK-SKX: # %bb.0: # %entry
; CHECK-SKX-NEXT: movl %edx, %eax
; CHECK-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-SKX-NEXT: xorl %ecx, %ecx
; CHECK-SKX-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-SKX-NEXT: .p2align 4, 0x90
; CHECK-SKX-NEXT: .LBB9_1: # %vector.body
; CHECK-SKX-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-SKX-NEXT: vpmovsxbw (%rdi,%rcx), %zmm2
; CHECK-SKX-NEXT: vpmovsxbw (%rsi,%rcx), %zmm3
; CHECK-SKX-NEXT: vpmaddwd %zmm2, %zmm3, %zmm2
; CHECK-SKX-NEXT: vpaddd %zmm1, %zmm2, %zmm1
; CHECK-SKX-NEXT: addq $32, %rcx
; CHECK-SKX-NEXT: cmpq %rcx, %rax
; CHECK-SKX-NEXT: jne .LBB9_1
; CHECK-SKX-NEXT: # %bb.2: # %middle.block
; CHECK-SKX-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; CHECK-SKX-NEXT: vextracti64x4 $1, %zmm0, %ymm1
; CHECK-SKX-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; CHECK-SKX-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vpsrlq $32, %xmm0, %xmm1
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vmovd %xmm0, %eax
; CHECK-SKX-NEXT: vzeroupper
; CHECK-SKX-NEXT: retq
;
; CHECK-AVX512-LABEL: _Z9test_charPcS_i_512:
; CHECK-AVX512: # %bb.0: # %entry
; CHECK-AVX512-NEXT: movl %edx, %eax
; CHECK-AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-AVX512-NEXT: xorl %ecx, %ecx
; CHECK-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-AVX512-NEXT: .p2align 4, 0x90
; CHECK-AVX512-NEXT: .LBB9_1: # %vector.body
; CHECK-AVX512-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-AVX512-NEXT: vpmovsxbw (%rdi,%rcx), %zmm2
; CHECK-AVX512-NEXT: vpmovsxbw (%rsi,%rcx), %zmm3
; CHECK-AVX512-NEXT: vpmaddwd %zmm2, %zmm3, %zmm2
; CHECK-AVX512-NEXT: vpaddd %zmm1, %zmm2, %zmm1
; CHECK-AVX512-NEXT: addq $32, %rcx
; CHECK-AVX512-NEXT: cmpq %rcx, %rax
; CHECK-AVX512-NEXT: jne .LBB9_1
; CHECK-AVX512-NEXT: # %bb.2: # %middle.block
; CHECK-AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; CHECK-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
; CHECK-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; CHECK-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vpsrlq $32, %xmm0, %xmm1
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vmovd %xmm0, %eax
; CHECK-AVX512-NEXT: vzeroupper
; CHECK-AVX512-NEXT: retq
;
; CHECK-VBMI-LABEL: _Z9test_charPcS_i_512:
; CHECK-VBMI: # %bb.0: # %entry
; CHECK-VBMI-NEXT: movl %edx, %eax
; CHECK-VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-VBMI-NEXT: xorl %ecx, %ecx
; CHECK-VBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-VBMI-NEXT: .p2align 4, 0x90
; CHECK-VBMI-NEXT: .LBB9_1: # %vector.body
; CHECK-VBMI-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-VBMI-NEXT: vpmovsxbw (%rdi,%rcx), %zmm2
; CHECK-VBMI-NEXT: vpmovsxbw (%rsi,%rcx), %zmm3
; CHECK-VBMI-NEXT: vpmaddwd %zmm2, %zmm3, %zmm2
; CHECK-VBMI-NEXT: vpaddd %zmm1, %zmm2, %zmm1
; CHECK-VBMI-NEXT: addq $32, %rcx
; CHECK-VBMI-NEXT: cmpq %rcx, %rax
; CHECK-VBMI-NEXT: jne .LBB9_1
; CHECK-VBMI-NEXT: # %bb.2: # %middle.block
; CHECK-VBMI-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; CHECK-VBMI-NEXT: vextracti64x4 $1, %zmm0, %ymm1
; CHECK-VBMI-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; CHECK-VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vmovd %xmm0, %eax
; CHECK-VBMI-NEXT: vzeroupper
; CHECK-VBMI-NEXT: retq
entry:
%3 = zext i32 %2 to i64
br label %vector.body
vector.body:
%index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
%vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
%4 = getelementptr inbounds i8, i8* %0, i64 %index
%5 = bitcast i8* %4 to <32 x i8>*
%wide.load = load <32 x i8>, <32 x i8>* %5, align 1
%6 = sext <32 x i8> %wide.load to <32 x i32>
%7 = getelementptr inbounds i8, i8* %1, i64 %index
%8 = bitcast i8* %7 to <32 x i8>*
%wide.load14 = load <32 x i8>, <32 x i8>* %8, align 1
%9 = sext <32 x i8> %wide.load14 to <32 x i32>
%10 = mul nsw <32 x i32> %9, %6
%11 = add nsw <32 x i32> %10, %vec.phi
%index.next = add i64 %index, 32
%12 = icmp eq i64 %index.next, %3
br i1 %12, label %middle.block, label %vector.body
middle.block:
%rdx.shuf1 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx1 = add <32 x i32> %11, %rdx.shuf1
%rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf
%rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx32 = add <32 x i32> %bin.rdx, %rdx.shuf15
%rdx.shuf17 = shufflevector <32 x i32> %bin.rdx32, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx18 = add <32 x i32> %bin.rdx32, %rdx.shuf17
%rdx.shuf19 = shufflevector <32 x i32> %bin.rdx18, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx20 = add <32 x i32> %bin.rdx18, %rdx.shuf19
%13 = extractelement <32 x i32> %bin.rdx20, i32 0
ret i32 %13
}
@a = dso_local global [1024 x i8] zeroinitializer, align 16
@b = dso_local global [1024 x i8] zeroinitializer, align 16
define dso_local i32 @sad_16i8_256() "min-legal-vector-width"="256" {
; CHECK-SKX-LABEL: sad_16i8_256:
; CHECK-SKX: # %bb.0: # %entry
; CHECK-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-SKX-NEXT: movq $-1024, %rax # imm = 0xFC00
; CHECK-SKX-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-SKX-NEXT: .p2align 4, 0x90
; CHECK-SKX-NEXT: .LBB10_1: # %vector.body
; CHECK-SKX-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-SKX-NEXT: vmovdqu a+1024(%rax), %xmm2
; CHECK-SKX-NEXT: vpsadbw b+1024(%rax), %xmm2, %xmm2
; CHECK-SKX-NEXT: vpaddd %ymm1, %ymm2, %ymm1
; CHECK-SKX-NEXT: addq $4, %rax
; CHECK-SKX-NEXT: jne .LBB10_1
; CHECK-SKX-NEXT: # %bb.2: # %middle.block
; CHECK-SKX-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; CHECK-SKX-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vpsrlq $32, %xmm0, %xmm1
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vmovd %xmm0, %eax
; CHECK-SKX-NEXT: vzeroupper
; CHECK-SKX-NEXT: retq
;
; CHECK-AVX512-LABEL: sad_16i8_256:
; CHECK-AVX512: # %bb.0: # %entry
; CHECK-AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-AVX512-NEXT: movq $-1024, %rax # imm = 0xFC00
; CHECK-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-AVX512-NEXT: .p2align 4, 0x90
; CHECK-AVX512-NEXT: .LBB10_1: # %vector.body
; CHECK-AVX512-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-AVX512-NEXT: vmovdqu a+1024(%rax), %xmm2
; CHECK-AVX512-NEXT: vpsadbw b+1024(%rax), %xmm2, %xmm2
; CHECK-AVX512-NEXT: vpaddd %ymm1, %ymm2, %ymm1
; CHECK-AVX512-NEXT: addq $4, %rax
; CHECK-AVX512-NEXT: jne .LBB10_1
; CHECK-AVX512-NEXT: # %bb.2: # %middle.block
; CHECK-AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; CHECK-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vpsrlq $32, %xmm0, %xmm1
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vmovd %xmm0, %eax
; CHECK-AVX512-NEXT: vzeroupper
; CHECK-AVX512-NEXT: retq
;
; CHECK-VBMI-LABEL: sad_16i8_256:
; CHECK-VBMI: # %bb.0: # %entry
; CHECK-VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-VBMI-NEXT: movq $-1024, %rax # imm = 0xFC00
; CHECK-VBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-VBMI-NEXT: .p2align 4, 0x90
; CHECK-VBMI-NEXT: .LBB10_1: # %vector.body
; CHECK-VBMI-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-VBMI-NEXT: vmovdqu a+1024(%rax), %xmm2
; CHECK-VBMI-NEXT: vpsadbw b+1024(%rax), %xmm2, %xmm2
; CHECK-VBMI-NEXT: vpaddd %ymm1, %ymm2, %ymm1
; CHECK-VBMI-NEXT: addq $4, %rax
; CHECK-VBMI-NEXT: jne .LBB10_1
; CHECK-VBMI-NEXT: # %bb.2: # %middle.block
; CHECK-VBMI-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; CHECK-VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vmovd %xmm0, %eax
; CHECK-VBMI-NEXT: vzeroupper
; CHECK-VBMI-NEXT: retq
entry:
br label %vector.body
vector.body:
%index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
%vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ]
%0 = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %index
%1 = bitcast i8* %0 to <16 x i8>*
%wide.load = load <16 x i8>, <16 x i8>* %1, align 4
%2 = zext <16 x i8> %wide.load to <16 x i32>
%3 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %index
%4 = bitcast i8* %3 to <16 x i8>*
%wide.load1 = load <16 x i8>, <16 x i8>* %4, align 4
%5 = zext <16 x i8> %wide.load1 to <16 x i32>
%6 = sub nsw <16 x i32> %2, %5
%7 = icmp sgt <16 x i32> %6, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
%8 = sub nsw <16 x i32> zeroinitializer, %6
%9 = select <16 x i1> %7, <16 x i32> %6, <16 x i32> %8
%10 = add nsw <16 x i32> %9, %vec.phi
%index.next = add i64 %index, 4
%11 = icmp eq i64 %index.next, 1024
br i1 %11, label %middle.block, label %vector.body
middle.block:
%rdx.shuf = shufflevector <16 x i32> %10, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx = add <16 x i32> %10, %rdx.shuf
%rdx.shuf2 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx2 = add <16 x i32> %bin.rdx, %rdx.shuf2
%rdx.shuf3 = shufflevector <16 x i32> %bin.rdx2, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx3 = add <16 x i32> %bin.rdx2, %rdx.shuf3
%rdx.shuf4 = shufflevector <16 x i32> %bin.rdx3, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx4 = add <16 x i32> %bin.rdx3, %rdx.shuf4
%12 = extractelement <16 x i32> %bin.rdx4, i32 0
ret i32 %12
}
define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" {
; CHECK-SKX-LABEL: sad_16i8_512:
; CHECK-SKX: # %bb.0: # %entry
; CHECK-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-SKX-NEXT: movq $-1024, %rax # imm = 0xFC00
; CHECK-SKX-NEXT: .p2align 4, 0x90
; CHECK-SKX-NEXT: .LBB11_1: # %vector.body
; CHECK-SKX-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-SKX-NEXT: vmovdqu a+1024(%rax), %xmm1
; CHECK-SKX-NEXT: vpsadbw b+1024(%rax), %xmm1, %xmm1
; CHECK-SKX-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; CHECK-SKX-NEXT: addq $4, %rax
; CHECK-SKX-NEXT: jne .LBB11_1
; CHECK-SKX-NEXT: # %bb.2: # %middle.block
; CHECK-SKX-NEXT: vextracti64x4 $1, %zmm0, %ymm1
; CHECK-SKX-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; CHECK-SKX-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vpsrlq $32, %xmm0, %xmm1
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vmovd %xmm0, %eax
; CHECK-SKX-NEXT: vzeroupper
; CHECK-SKX-NEXT: retq
;
; CHECK-AVX512-LABEL: sad_16i8_512:
; CHECK-AVX512: # %bb.0: # %entry
; CHECK-AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-AVX512-NEXT: movq $-1024, %rax # imm = 0xFC00
; CHECK-AVX512-NEXT: .p2align 4, 0x90
; CHECK-AVX512-NEXT: .LBB11_1: # %vector.body
; CHECK-AVX512-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-AVX512-NEXT: vmovdqu a+1024(%rax), %xmm1
; CHECK-AVX512-NEXT: vpsadbw b+1024(%rax), %xmm1, %xmm1
; CHECK-AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; CHECK-AVX512-NEXT: addq $4, %rax
; CHECK-AVX512-NEXT: jne .LBB11_1
; CHECK-AVX512-NEXT: # %bb.2: # %middle.block
; CHECK-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
; CHECK-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; CHECK-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vpsrlq $32, %xmm0, %xmm1
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vmovd %xmm0, %eax
; CHECK-AVX512-NEXT: vzeroupper
; CHECK-AVX512-NEXT: retq
;
; CHECK-VBMI-LABEL: sad_16i8_512:
; CHECK-VBMI: # %bb.0: # %entry
; CHECK-VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-VBMI-NEXT: movq $-1024, %rax # imm = 0xFC00
; CHECK-VBMI-NEXT: .p2align 4, 0x90
; CHECK-VBMI-NEXT: .LBB11_1: # %vector.body
; CHECK-VBMI-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-VBMI-NEXT: vmovdqu a+1024(%rax), %xmm1
; CHECK-VBMI-NEXT: vpsadbw b+1024(%rax), %xmm1, %xmm1
; CHECK-VBMI-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; CHECK-VBMI-NEXT: addq $4, %rax
; CHECK-VBMI-NEXT: jne .LBB11_1
; CHECK-VBMI-NEXT: # %bb.2: # %middle.block
; CHECK-VBMI-NEXT: vextracti64x4 $1, %zmm0, %ymm1
; CHECK-VBMI-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; CHECK-VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vmovd %xmm0, %eax
; CHECK-VBMI-NEXT: vzeroupper
; CHECK-VBMI-NEXT: retq
entry:
br label %vector.body
vector.body:
%index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
%vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ]
%0 = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %index
%1 = bitcast i8* %0 to <16 x i8>*
%wide.load = load <16 x i8>, <16 x i8>* %1, align 4
%2 = zext <16 x i8> %wide.load to <16 x i32>
%3 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %index
%4 = bitcast i8* %3 to <16 x i8>*
%wide.load1 = load <16 x i8>, <16 x i8>* %4, align 4
%5 = zext <16 x i8> %wide.load1 to <16 x i32>
%6 = sub nsw <16 x i32> %2, %5
%7 = icmp sgt <16 x i32> %6, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
%8 = sub nsw <16 x i32> zeroinitializer, %6
%9 = select <16 x i1> %7, <16 x i32> %6, <16 x i32> %8
%10 = add nsw <16 x i32> %9, %vec.phi
%index.next = add i64 %index, 4
%11 = icmp eq i64 %index.next, 1024
br i1 %11, label %middle.block, label %vector.body
middle.block:
%rdx.shuf = shufflevector <16 x i32> %10, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx = add <16 x i32> %10, %rdx.shuf
%rdx.shuf2 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx2 = add <16 x i32> %bin.rdx, %rdx.shuf2
%rdx.shuf3 = shufflevector <16 x i32> %bin.rdx2, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx3 = add <16 x i32> %bin.rdx2, %rdx.shuf3
%rdx.shuf4 = shufflevector <16 x i32> %bin.rdx3, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%bin.rdx4 = add <16 x i32> %bin.rdx3, %rdx.shuf4
%12 = extractelement <16 x i32> %bin.rdx4, i32 0
ret i32 %12
}
define dso_local void @sbto16f32_256(<16 x i16> %a, <16 x float>* %res) "min-legal-vector-width"="256" {
; CHECK-LABEL: sbto16f32_256:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovw2m %ymm0, %k0
; CHECK-NEXT: kshiftrw $8, %k0, %k1
; CHECK-NEXT: vpmovm2d %k1, %ymm0
; CHECK-NEXT: vcvtdq2ps %ymm0, %ymm0
; CHECK-NEXT: vpmovm2d %k0, %ymm1
; CHECK-NEXT: vcvtdq2ps %ymm1, %ymm1
; CHECK-NEXT: vmovaps %ymm1, (%rdi)
; CHECK-NEXT: vmovaps %ymm0, 32(%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%mask = icmp slt <16 x i16> %a, zeroinitializer
%1 = sitofp <16 x i1> %mask to <16 x float>
store <16 x float> %1, <16 x float>* %res
ret void
}
define dso_local void @sbto16f32_512(<16 x i16> %a, <16 x float>* %res) "min-legal-vector-width"="512" {
; CHECK-LABEL: sbto16f32_512:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovw2m %ymm0, %k0
; CHECK-NEXT: vpmovm2d %k0, %zmm0
; CHECK-NEXT: vcvtdq2ps %zmm0, %zmm0
; CHECK-NEXT: vmovaps %zmm0, (%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%mask = icmp slt <16 x i16> %a, zeroinitializer
%1 = sitofp <16 x i1> %mask to <16 x float>
store <16 x float> %1, <16 x float>* %res
ret void
}
define dso_local void @sbto16f64_256(<16 x i16> %a, <16 x double>* %res) "min-legal-vector-width"="256" {
; CHECK-LABEL: sbto16f64_256:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovw2m %ymm0, %k0
; CHECK-NEXT: kshiftrw $8, %k0, %k1
; CHECK-NEXT: vpmovm2d %k1, %ymm0
; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm1
; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm0
; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm0
; CHECK-NEXT: vpmovm2d %k0, %ymm2
; CHECK-NEXT: vcvtdq2pd %xmm2, %ymm3
; CHECK-NEXT: vextracti128 $1, %ymm2, %xmm2
; CHECK-NEXT: vcvtdq2pd %xmm2, %ymm2
; CHECK-NEXT: vmovaps %ymm2, 32(%rdi)
; CHECK-NEXT: vmovaps %ymm3, (%rdi)
; CHECK-NEXT: vmovaps %ymm0, 96(%rdi)
; CHECK-NEXT: vmovaps %ymm1, 64(%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%mask = icmp slt <16 x i16> %a, zeroinitializer
%1 = sitofp <16 x i1> %mask to <16 x double>
store <16 x double> %1, <16 x double>* %res
ret void
}
define dso_local void @sbto16f64_512(<16 x i16> %a, <16 x double>* %res) "min-legal-vector-width"="512" {
; CHECK-LABEL: sbto16f64_512:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovw2m %ymm0, %k0
; CHECK-NEXT: vpmovm2d %k0, %zmm0
; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm1
; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm0
; CHECK-NEXT: vmovaps %zmm0, 64(%rdi)
; CHECK-NEXT: vmovaps %zmm1, (%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%mask = icmp slt <16 x i16> %a, zeroinitializer
%1 = sitofp <16 x i1> %mask to <16 x double>
store <16 x double> %1, <16 x double>* %res
ret void
}
define dso_local void @ubto16f32_256(<16 x i16> %a, <16 x float>* %res) "min-legal-vector-width"="256" {
; CHECK-LABEL: ubto16f32_256:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovw2m %ymm0, %k0
; CHECK-NEXT: kshiftrw $8, %k0, %k1
; CHECK-NEXT: vpmovm2d %k1, %ymm0
; CHECK-NEXT: vpsrld $31, %ymm0, %ymm0
; CHECK-NEXT: vcvtdq2ps %ymm0, %ymm0
; CHECK-NEXT: vpmovm2d %k0, %ymm1
; CHECK-NEXT: vpsrld $31, %ymm1, %ymm1
; CHECK-NEXT: vcvtdq2ps %ymm1, %ymm1
; CHECK-NEXT: vmovaps %ymm1, (%rdi)
; CHECK-NEXT: vmovaps %ymm0, 32(%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%mask = icmp slt <16 x i16> %a, zeroinitializer
%1 = uitofp <16 x i1> %mask to <16 x float>
store <16 x float> %1, <16 x float>* %res
ret void
}
define dso_local void @ubto16f32_512(<16 x i16> %a, <16 x float>* %res) "min-legal-vector-width"="512" {
; CHECK-LABEL: ubto16f32_512:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovw2m %ymm0, %k0
; CHECK-NEXT: vpmovm2d %k0, %zmm0
; CHECK-NEXT: vpsrld $31, %zmm0, %zmm0
; CHECK-NEXT: vcvtdq2ps %zmm0, %zmm0
; CHECK-NEXT: vmovaps %zmm0, (%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%mask = icmp slt <16 x i16> %a, zeroinitializer
%1 = uitofp <16 x i1> %mask to <16 x float>
store <16 x float> %1, <16 x float>* %res
ret void
}
define dso_local void @ubto16f64_256(<16 x i16> %a, <16 x double>* %res) "min-legal-vector-width"="256" {
; CHECK-LABEL: ubto16f64_256:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovw2m %ymm0, %k0
; CHECK-NEXT: kshiftrw $8, %k0, %k1
; CHECK-NEXT: vpmovm2d %k1, %ymm0
; CHECK-NEXT: vpsrld $31, %ymm0, %ymm0
; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm1
; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm0
; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm0
; CHECK-NEXT: vpmovm2d %k0, %ymm2
; CHECK-NEXT: vpsrld $31, %ymm2, %ymm2
; CHECK-NEXT: vcvtdq2pd %xmm2, %ymm3
; CHECK-NEXT: vextracti128 $1, %ymm2, %xmm2
; CHECK-NEXT: vcvtdq2pd %xmm2, %ymm2
; CHECK-NEXT: vmovaps %ymm2, 32(%rdi)
; CHECK-NEXT: vmovaps %ymm3, (%rdi)
; CHECK-NEXT: vmovaps %ymm0, 96(%rdi)
; CHECK-NEXT: vmovaps %ymm1, 64(%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%mask = icmp slt <16 x i16> %a, zeroinitializer
%1 = uitofp <16 x i1> %mask to <16 x double>
store <16 x double> %1, <16 x double>* %res
ret void
}
define dso_local void @ubto16f64_512(<16 x i16> %a, <16 x double>* %res) "min-legal-vector-width"="512" {
; CHECK-LABEL: ubto16f64_512:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovw2m %ymm0, %k0
; CHECK-NEXT: vpmovm2d %k0, %zmm0
; CHECK-NEXT: vpsrld $31, %zmm0, %zmm0
; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm1
; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm0
; CHECK-NEXT: vmovaps %zmm0, 64(%rdi)
; CHECK-NEXT: vmovaps %zmm1, (%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%mask = icmp slt <16 x i16> %a, zeroinitializer
%1 = uitofp <16 x i1> %mask to <16 x double>
store <16 x double> %1, <16 x double>* %res
ret void
}
define <16 x i16> @test_16f32toub_256(<16 x float>* %ptr, <16 x i16> %passthru) "min-legal-vector-width"="256" {
; CHECK-LABEL: test_16f32toub_256:
; CHECK: # %bb.0:
; CHECK-NEXT: vcvttps2dq (%rdi), %ymm1
; CHECK-NEXT: vpslld $31, %ymm1, %ymm1
; CHECK-NEXT: vpmovd2m %ymm1, %k0
; CHECK-NEXT: vcvttps2dq 32(%rdi), %ymm1
; CHECK-NEXT: vpslld $31, %ymm1, %ymm1
; CHECK-NEXT: vpmovd2m %ymm1, %k1
; CHECK-NEXT: kunpckbw %k0, %k1, %k1
; CHECK-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%a = load <16 x float>, <16 x float>* %ptr
%mask = fptoui <16 x float> %a to <16 x i1>
%select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer
ret <16 x i16> %select
}
define <16 x i16> @test_16f32toub_512(<16 x float>* %ptr, <16 x i16> %passthru) "min-legal-vector-width"="512" {
; CHECK-LABEL: test_16f32toub_512:
; CHECK: # %bb.0:
; CHECK-NEXT: vcvttps2dq (%rdi), %zmm1
; CHECK-NEXT: vpslld $31, %zmm1, %zmm1
; CHECK-NEXT: vpmovd2m %zmm1, %k1
; CHECK-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%a = load <16 x float>, <16 x float>* %ptr
%mask = fptoui <16 x float> %a to <16 x i1>
%select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer
ret <16 x i16> %select
}
define <16 x i16> @test_16f32tosb_256(<16 x float>* %ptr, <16 x i16> %passthru) "min-legal-vector-width"="256" {
; CHECK-LABEL: test_16f32tosb_256:
; CHECK: # %bb.0:
; CHECK-NEXT: vcvttps2dq (%rdi), %ymm1
; CHECK-NEXT: vpmovd2m %ymm1, %k0
; CHECK-NEXT: vcvttps2dq 32(%rdi), %ymm1
; CHECK-NEXT: vpmovd2m %ymm1, %k1
; CHECK-NEXT: kunpckbw %k0, %k1, %k1
; CHECK-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%a = load <16 x float>, <16 x float>* %ptr
%mask = fptosi <16 x float> %a to <16 x i1>
%select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer
ret <16 x i16> %select
}
define <16 x i16> @test_16f32tosb_512(<16 x float>* %ptr, <16 x i16> %passthru) "min-legal-vector-width"="512" {
; CHECK-LABEL: test_16f32tosb_512:
; CHECK: # %bb.0:
; CHECK-NEXT: vcvttps2dq (%rdi), %zmm1
; CHECK-NEXT: vpmovd2m %zmm1, %k1
; CHECK-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z}
; CHECK-NEXT: retq
%a = load <16 x float>, <16 x float>* %ptr
%mask = fptosi <16 x float> %a to <16 x i1>
%select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer
ret <16 x i16> %select
}
define dso_local void @mul256(<64 x i8>* %a, <64 x i8>* %b, <64 x i8>* %c) "min-legal-vector-width"="256" {
; CHECK-SKX-VBMI-LABEL: mul256:
; CHECK-SKX-VBMI: # %bb.0:
; CHECK-SKX-VBMI-NEXT: vmovdqa (%rdi), %ymm0
; CHECK-SKX-VBMI-NEXT: vmovdqa 32(%rdi), %ymm1
; CHECK-SKX-VBMI-NEXT: vmovdqa (%rsi), %ymm2
; CHECK-SKX-VBMI-NEXT: vmovdqa 32(%rsi), %ymm3
; CHECK-SKX-VBMI-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-SKX-VBMI-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-SKX-VBMI-NEXT: vpmullw %ymm4, %ymm5, %ymm4
; CHECK-SKX-VBMI-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-SKX-VBMI-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-SKX-VBMI-NEXT: vpmullw %ymm3, %ymm1, %ymm1
; CHECK-SKX-VBMI-NEXT: vmovdqa {{.*#+}} ymm3 = [0,2,4,6,8,10,12,14,32,34,36,38,40,42,44,46,16,18,20,22,24,26,28,30,48,50,52,54,56,58,60,62]
; CHECK-SKX-VBMI-NEXT: vpermt2b %ymm4, %ymm3, %ymm1
; CHECK-SKX-VBMI-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-SKX-VBMI-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-SKX-VBMI-NEXT: vpmullw %ymm4, %ymm5, %ymm4
; CHECK-SKX-VBMI-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-SKX-VBMI-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-SKX-VBMI-NEXT: vpmullw %ymm2, %ymm0, %ymm0
; CHECK-SKX-VBMI-NEXT: vpermt2b %ymm4, %ymm3, %ymm0
; CHECK-SKX-VBMI-NEXT: vmovdqa %ymm0, (%rdx)
; CHECK-SKX-VBMI-NEXT: vmovdqa %ymm1, 32(%rdx)
; CHECK-SKX-VBMI-NEXT: vzeroupper
; CHECK-SKX-VBMI-NEXT: retq
;
; CHECK-AVX512-LABEL: mul256:
; CHECK-AVX512: # %bb.0:
; CHECK-AVX512-NEXT: vmovdqa (%rdi), %ymm0
; CHECK-AVX512-NEXT: vmovdqa 32(%rdi), %ymm1
; CHECK-AVX512-NEXT: vmovdqa (%rsi), %ymm2
; CHECK-AVX512-NEXT: vmovdqa 32(%rsi), %ymm3
; CHECK-AVX512-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-AVX512-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-AVX512-NEXT: vpmullw %ymm4, %ymm5, %ymm4
; CHECK-AVX512-NEXT: vpbroadcastw {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; CHECK-AVX512-NEXT: vpand %ymm5, %ymm4, %ymm4
; CHECK-AVX512-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-AVX512-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-AVX512-NEXT: vpmullw %ymm3, %ymm1, %ymm1
; CHECK-AVX512-NEXT: vpand %ymm5, %ymm1, %ymm1
; CHECK-AVX512-NEXT: vpackuswb %ymm4, %ymm1, %ymm1
; CHECK-AVX512-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-AVX512-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-AVX512-NEXT: vpmullw %ymm3, %ymm4, %ymm3
; CHECK-AVX512-NEXT: vpand %ymm5, %ymm3, %ymm3
; CHECK-AVX512-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-AVX512-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-AVX512-NEXT: vpmullw %ymm2, %ymm0, %ymm0
; CHECK-AVX512-NEXT: vpand %ymm5, %ymm0, %ymm0
; CHECK-AVX512-NEXT: vpackuswb %ymm3, %ymm0, %ymm0
; CHECK-AVX512-NEXT: vmovdqa %ymm0, (%rdx)
; CHECK-AVX512-NEXT: vmovdqa %ymm1, 32(%rdx)
; CHECK-AVX512-NEXT: vzeroupper
; CHECK-AVX512-NEXT: retq
;
; CHECK-VBMI-LABEL: mul256:
; CHECK-VBMI: # %bb.0:
; CHECK-VBMI-NEXT: vmovdqa (%rdi), %ymm0
; CHECK-VBMI-NEXT: vmovdqa 32(%rdi), %ymm1
; CHECK-VBMI-NEXT: vmovdqa (%rsi), %ymm2
; CHECK-VBMI-NEXT: vmovdqa 32(%rsi), %ymm3
; CHECK-VBMI-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-VBMI-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-VBMI-NEXT: vpmullw %ymm4, %ymm5, %ymm4
; CHECK-VBMI-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-VBMI-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-VBMI-NEXT: vpmullw %ymm3, %ymm1, %ymm1
; CHECK-VBMI-NEXT: vmovdqa {{.*#+}} ymm3 = [0,2,4,6,8,10,12,14,32,34,36,38,40,42,44,46,16,18,20,22,24,26,28,30,48,50,52,54,56,58,60,62]
; CHECK-VBMI-NEXT: vpermt2b %ymm4, %ymm3, %ymm1
; CHECK-VBMI-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-VBMI-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-VBMI-NEXT: vpmullw %ymm4, %ymm5, %ymm4
; CHECK-VBMI-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-VBMI-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-VBMI-NEXT: vpmullw %ymm2, %ymm0, %ymm0
; CHECK-VBMI-NEXT: vpermt2b %ymm4, %ymm3, %ymm0
; CHECK-VBMI-NEXT: vmovdqa %ymm0, (%rdx)
; CHECK-VBMI-NEXT: vmovdqa %ymm1, 32(%rdx)
; CHECK-VBMI-NEXT: vzeroupper
; CHECK-VBMI-NEXT: retq
%d = load <64 x i8>, <64 x i8>* %a
%e = load <64 x i8>, <64 x i8>* %b
%f = mul <64 x i8> %d, %e
store <64 x i8> %f, <64 x i8>* %c
ret void
}
define dso_local void @mul512(<64 x i8>* %a, <64 x i8>* %b, <64 x i8>* %c) "min-legal-vector-width"="512" {
; CHECK-SKX-VBMI-LABEL: mul512:
; CHECK-SKX-VBMI: # %bb.0:
; CHECK-SKX-VBMI-NEXT: vmovdqa64 (%rdi), %zmm0
; CHECK-SKX-VBMI-NEXT: vmovdqa64 (%rsi), %zmm1
; CHECK-SKX-VBMI-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; CHECK-SKX-VBMI-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; CHECK-SKX-VBMI-NEXT: vpmullw %zmm2, %zmm3, %zmm2
; CHECK-SKX-VBMI-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
; CHECK-SKX-VBMI-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
; CHECK-SKX-VBMI-NEXT: vpmullw %zmm1, %zmm0, %zmm0
; CHECK-SKX-VBMI-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,2,4,6,8,10,12,14,64,66,68,70,72,74,76,78,16,18,20,22,24,26,28,30,80,82,84,86,88,90,92,94,32,34,36,38,40,42,44,46,96,98,100,102,104,106,108,110,48,50,52,54,56,58,60,62,112,114,116,118,120,122,124,126]
; CHECK-SKX-VBMI-NEXT: vpermi2b %zmm2, %zmm0, %zmm1
; CHECK-SKX-VBMI-NEXT: vmovdqa64 %zmm1, (%rdx)
; CHECK-SKX-VBMI-NEXT: vzeroupper
; CHECK-SKX-VBMI-NEXT: retq
;
; CHECK-AVX512-LABEL: mul512:
; CHECK-AVX512: # %bb.0:
; CHECK-AVX512-NEXT: vmovdqa64 (%rdi), %zmm0
; CHECK-AVX512-NEXT: vmovdqa64 (%rsi), %zmm1
; CHECK-AVX512-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; CHECK-AVX512-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; CHECK-AVX512-NEXT: vpmullw %zmm2, %zmm3, %zmm2
; CHECK-AVX512-NEXT: vpbroadcastw {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; CHECK-AVX512-NEXT: vpandq %zmm3, %zmm2, %zmm2
; CHECK-AVX512-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
; CHECK-AVX512-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
; CHECK-AVX512-NEXT: vpmullw %zmm1, %zmm0, %zmm0
; CHECK-AVX512-NEXT: vpandq %zmm3, %zmm0, %zmm0
; CHECK-AVX512-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
; CHECK-AVX512-NEXT: vmovdqa64 %zmm0, (%rdx)
; CHECK-AVX512-NEXT: vzeroupper
; CHECK-AVX512-NEXT: retq
;
; CHECK-VBMI-LABEL: mul512:
; CHECK-VBMI: # %bb.0:
; CHECK-VBMI-NEXT: vmovdqa64 (%rdi), %zmm0
; CHECK-VBMI-NEXT: vmovdqa64 (%rsi), %zmm1
; CHECK-VBMI-NEXT: vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; CHECK-VBMI-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; CHECK-VBMI-NEXT: vpmullw %zmm2, %zmm3, %zmm2
; CHECK-VBMI-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
; CHECK-VBMI-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
; CHECK-VBMI-NEXT: vpmullw %zmm1, %zmm0, %zmm0
; CHECK-VBMI-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,2,4,6,8,10,12,14,64,66,68,70,72,74,76,78,16,18,20,22,24,26,28,30,80,82,84,86,88,90,92,94,32,34,36,38,40,42,44,46,96,98,100,102,104,106,108,110,48,50,52,54,56,58,60,62,112,114,116,118,120,122,124,126]
; CHECK-VBMI-NEXT: vpermi2b %zmm2, %zmm0, %zmm1
; CHECK-VBMI-NEXT: vmovdqa64 %zmm1, (%rdx)
; CHECK-VBMI-NEXT: vzeroupper
; CHECK-VBMI-NEXT: retq
%d = load <64 x i8>, <64 x i8>* %a
%e = load <64 x i8>, <64 x i8>* %b
%f = mul <64 x i8> %d, %e
store <64 x i8> %f, <64 x i8>* %c
ret void
}
; This threw an assertion at one point.
define <4 x i32> @mload_v4i32(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %dst) "min-legal-vector-width"="256" {
; CHECK-LABEL: mload_v4i32:
; CHECK: # %bb.0:
; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k1
; CHECK-NEXT: vpblendmd (%rdi), %xmm1, %xmm0 {%k1}
; CHECK-NEXT: retq
%mask = icmp eq <4 x i32> %trigger, zeroinitializer
%res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
ret <4 x i32> %res
}
declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)
define <16 x i32> @trunc_v16i64_v16i32(<16 x i64>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i64_v16i32:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rdi), %ymm0
; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT: vmovdqa 64(%rdi), %ymm2
; CHECK-NEXT: vmovdqa 96(%rdi), %ymm3
; CHECK-NEXT: vpmovqd %ymm0, %xmm0
; CHECK-NEXT: vpmovqd %ymm1, %xmm1
; CHECK-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; CHECK-NEXT: vpmovqd %ymm2, %xmm1
; CHECK-NEXT: vpmovqd %ymm3, %xmm2
; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; CHECK-NEXT: retq
%a = load <16 x i64>, <16 x i64>* %x
%b = trunc <16 x i64> %a to <16 x i32>
ret <16 x i32> %b
}
define <16 x i8> @trunc_v16i64_v16i8(<16 x i64>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i64_v16i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rdi), %ymm0
; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT: vmovdqa 64(%rdi), %ymm2
; CHECK-NEXT: vmovdqa 96(%rdi), %ymm3
; CHECK-NEXT: vpmovqb %ymm3, %xmm3
; CHECK-NEXT: vpmovqb %ymm2, %xmm2
; CHECK-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; CHECK-NEXT: vpmovqb %ymm1, %xmm1
; CHECK-NEXT: vpmovqb %ymm0, %xmm0
; CHECK-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%a = load <16 x i64>, <16 x i64>* %x
%b = trunc <16 x i64> %a to <16 x i8>
ret <16 x i8> %b
}
define <16 x i8> @trunc_v16i32_v16i8(<16 x i32>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i32_v16i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rdi), %ymm0
; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT: vpmovdb %ymm1, %xmm1
; CHECK-NEXT: vpmovdb %ymm0, %xmm0
; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%a = load <16 x i32>, <16 x i32>* %x
%b = trunc <16 x i32> %a to <16 x i8>
ret <16 x i8> %b
}
define <8 x i8> @trunc_v8i64_v8i8(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v8i64_v8i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rdi), %ymm0
; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT: vpmovqb %ymm1, %xmm1
; CHECK-NEXT: vpmovqb %ymm0, %xmm0
; CHECK-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%a = load <8 x i64>, <8 x i64>* %x
%b = trunc <8 x i64> %a to <8 x i8>
ret <8 x i8> %b
}
define <8 x i16> @trunc_v8i64_v8i16(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v8i64_v8i16:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rdi), %ymm0
; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT: vpmovqw %ymm1, %xmm1
; CHECK-NEXT: vpmovqw %ymm0, %xmm0
; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%a = load <8 x i64>, <8 x i64>* %x
%b = trunc <8 x i64> %a to <8 x i16>
ret <8 x i16> %b
}
define <8 x i32> @trunc_v8i64_v8i32_zeroes(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v8i64_v8i32_zeroes:
; CHECK: # %bb.0:
; CHECK-NEXT: vpsrlq $48, 32(%rdi), %ymm0
; CHECK-NEXT: vpsrlq $48, (%rdi), %ymm1
; CHECK-NEXT: vpackusdw %ymm0, %ymm1, %ymm0
; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-NEXT: retq
%a = load <8 x i64>, <8 x i64>* %x
%b = lshr <8 x i64> %a, <i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48>
%c = trunc <8 x i64> %b to <8 x i32>
ret <8 x i32> %c
}
define <16 x i16> @trunc_v16i32_v16i16_zeroes(<16 x i32>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i32_v16i16_zeroes:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rdi), %ymm1
; CHECK-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
; CHECK-NEXT: vpermi2w 32(%rdi), %ymm1, %ymm0
; CHECK-NEXT: retq
%a = load <16 x i32>, <16 x i32>* %x
%b = lshr <16 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
%c = trunc <16 x i32> %b to <16 x i16>
ret <16 x i16> %c
}
define <32 x i8> @trunc_v32i16_v32i8_zeroes(<32 x i16>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-SKX-VBMI-LABEL: trunc_v32i16_v32i8_zeroes:
; CHECK-SKX-VBMI: # %bb.0:
; CHECK-SKX-VBMI-NEXT: vmovdqa (%rdi), %ymm1
; CHECK-SKX-VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
; CHECK-SKX-VBMI-NEXT: vpermi2b 32(%rdi), %ymm1, %ymm0
; CHECK-SKX-VBMI-NEXT: retq
;
; CHECK-AVX512-LABEL: trunc_v32i16_v32i8_zeroes:
; CHECK-AVX512: # %bb.0:
; CHECK-AVX512-NEXT: vpsrlw $8, 32(%rdi), %ymm0
; CHECK-AVX512-NEXT: vpsrlw $8, (%rdi), %ymm1
; CHECK-AVX512-NEXT: vpackuswb %ymm0, %ymm1, %ymm0
; CHECK-AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-AVX512-NEXT: retq
;
; CHECK-VBMI-LABEL: trunc_v32i16_v32i8_zeroes:
; CHECK-VBMI: # %bb.0:
; CHECK-VBMI-NEXT: vmovdqa (%rdi), %ymm1
; CHECK-VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
; CHECK-VBMI-NEXT: vpermi2b 32(%rdi), %ymm1, %ymm0
; CHECK-VBMI-NEXT: retq
%a = load <32 x i16>, <32 x i16>* %x
%b = lshr <32 x i16> %a, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
%c = trunc <32 x i16> %b to <32 x i8>
ret <32 x i8> %c
}
define <8 x i32> @trunc_v8i64_v8i32_sign(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v8i64_v8i32_sign:
; CHECK: # %bb.0:
; CHECK-NEXT: vpsraq $48, 32(%rdi), %ymm0
; CHECK-NEXT: vpsraq $48, (%rdi), %ymm1
; CHECK-NEXT: vpmovqd %ymm1, %xmm1
; CHECK-NEXT: vpmovqd %ymm0, %xmm0
; CHECK-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; CHECK-NEXT: retq
%a = load <8 x i64>, <8 x i64>* %x
%b = ashr <8 x i64> %a, <i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48>
%c = trunc <8 x i64> %b to <8 x i32>
ret <8 x i32> %c
}
define <16 x i16> @trunc_v16i32_v16i16_sign(<16 x i32>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i32_v16i16_sign:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rdi), %ymm1
; CHECK-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
; CHECK-NEXT: vpermi2w 32(%rdi), %ymm1, %ymm0
; CHECK-NEXT: retq
%a = load <16 x i32>, <16 x i32>* %x
%b = ashr <16 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
%c = trunc <16 x i32> %b to <16 x i16>
ret <16 x i16> %c
}
define <32 x i8> @trunc_v32i16_v32i8_sign(<32 x i16>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-SKX-VBMI-LABEL: trunc_v32i16_v32i8_sign:
; CHECK-SKX-VBMI: # %bb.0:
; CHECK-SKX-VBMI-NEXT: vmovdqa (%rdi), %ymm1
; CHECK-SKX-VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
; CHECK-SKX-VBMI-NEXT: vpermi2b 32(%rdi), %ymm1, %ymm0
; CHECK-SKX-VBMI-NEXT: retq
;
; CHECK-AVX512-LABEL: trunc_v32i16_v32i8_sign:
; CHECK-AVX512: # %bb.0:
; CHECK-AVX512-NEXT: vpsrlw $8, 32(%rdi), %ymm0
; CHECK-AVX512-NEXT: vpsrlw $8, (%rdi), %ymm1
; CHECK-AVX512-NEXT: vpackuswb %ymm0, %ymm1, %ymm0
; CHECK-AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-AVX512-NEXT: retq
;
; CHECK-VBMI-LABEL: trunc_v32i16_v32i8_sign:
; CHECK-VBMI: # %bb.0:
; CHECK-VBMI-NEXT: vmovdqa (%rdi), %ymm1
; CHECK-VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
; CHECK-VBMI-NEXT: vpermi2b 32(%rdi), %ymm1, %ymm0
; CHECK-VBMI-NEXT: retq
%a = load <32 x i16>, <32 x i16>* %x
%b = ashr <32 x i16> %a, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
%c = trunc <32 x i16> %b to <32 x i8>
ret <32 x i8> %c
}
define dso_local void @zext_v16i8_v16i64(<16 x i8> %x, <16 x i64>* %y) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: zext_v16i8_v16i64:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
; CHECK-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; CHECK-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm1
; CHECK-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
; CHECK-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
; CHECK-NEXT: vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
; CHECK-NEXT: vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
; CHECK-NEXT: vmovdqa %ymm0, (%rdi)
; CHECK-NEXT: vmovdqa %ymm1, 64(%rdi)
; CHECK-NEXT: vmovdqa %ymm3, 96(%rdi)
; CHECK-NEXT: vmovdqa %ymm2, 32(%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%a = zext <16 x i8> %x to <16 x i64>
store <16 x i64> %a, <16 x i64>* %y
ret void
}
define dso_local void @sext_v16i8_v16i64(<16 x i8> %x, <16 x i64>* %y) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: sext_v16i8_v16i64:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovsxbw %xmm0, %ymm1
; CHECK-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; CHECK-NEXT: vpmovsxwq %xmm2, %ymm2
; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm1
; CHECK-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
; CHECK-NEXT: vpmovsxwq %xmm3, %ymm3
; CHECK-NEXT: vpmovsxwq %xmm1, %ymm1
; CHECK-NEXT: vpmovsxbq %xmm0, %ymm0
; CHECK-NEXT: vmovdqa %ymm0, (%rdi)
; CHECK-NEXT: vmovdqa %ymm1, 64(%rdi)
; CHECK-NEXT: vmovdqa %ymm3, 96(%rdi)
; CHECK-NEXT: vmovdqa %ymm2, 32(%rdi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%a = sext <16 x i8> %x to <16 x i64>
store <16 x i64> %a, <16 x i64>* %y
ret void
}
define dso_local void @vselect_split_v8i16_setcc(<8 x i16> %s, <8 x i16> %t, <8 x i64>* %p, <8 x i64>* %q, <8 x i64>* %r) "min-legal-vector-width"="256" {
; CHECK-LABEL: vselect_split_v8i16_setcc:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rsi), %ymm2
; CHECK-NEXT: vmovdqa 32(%rsi), %ymm3
; CHECK-NEXT: vpcmpeqw %xmm1, %xmm0, %k1
; CHECK-NEXT: kshiftrb $4, %k1, %k2
; CHECK-NEXT: vmovdqa64 32(%rdi), %ymm3 {%k2}
; CHECK-NEXT: vmovdqa64 (%rdi), %ymm2 {%k1}
; CHECK-NEXT: vmovdqa %ymm2, (%rdx)
; CHECK-NEXT: vmovdqa %ymm3, 32(%rdx)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%x = load <8 x i64>, <8 x i64>* %p
%y = load <8 x i64>, <8 x i64>* %q
%a = icmp eq <8 x i16> %s, %t
%b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y
store <8 x i64> %b, <8 x i64>* %r
ret void
}
define dso_local void @vselect_split_v8i32_setcc(<8 x i32> %s, <8 x i32> %t, <8 x i64>* %p, <8 x i64>* %q, <8 x i64>* %r) "min-legal-vector-width"="256" {
; CHECK-LABEL: vselect_split_v8i32_setcc:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rsi), %ymm2
; CHECK-NEXT: vmovdqa 32(%rsi), %ymm3
; CHECK-NEXT: vpcmpeqd %ymm1, %ymm0, %k1
; CHECK-NEXT: kshiftrb $4, %k1, %k2
; CHECK-NEXT: vmovdqa64 32(%rdi), %ymm3 {%k2}
; CHECK-NEXT: vmovdqa64 (%rdi), %ymm2 {%k1}
; CHECK-NEXT: vmovdqa %ymm2, (%rdx)
; CHECK-NEXT: vmovdqa %ymm3, 32(%rdx)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%x = load <8 x i64>, <8 x i64>* %p
%y = load <8 x i64>, <8 x i64>* %q
%a = icmp eq <8 x i32> %s, %t
%b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y
store <8 x i64> %b, <8 x i64>* %r
ret void
}
define dso_local void @vselect_split_v16i8_setcc(<16 x i8> %s, <16 x i8> %t, <16 x i32>* %p, <16 x i32>* %q, <16 x i32>* %r) "min-legal-vector-width"="256" {
; CHECK-LABEL: vselect_split_v16i8_setcc:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rsi), %ymm2
; CHECK-NEXT: vmovdqa 32(%rsi), %ymm3
; CHECK-NEXT: vpcmpeqb %xmm1, %xmm0, %k1
; CHECK-NEXT: kshiftrw $8, %k1, %k2
; CHECK-NEXT: vmovdqa32 32(%rdi), %ymm3 {%k2}
; CHECK-NEXT: vmovdqa32 (%rdi), %ymm2 {%k1}
; CHECK-NEXT: vmovdqa %ymm2, (%rdx)
; CHECK-NEXT: vmovdqa %ymm3, 32(%rdx)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%x = load <16 x i32>, <16 x i32>* %p
%y = load <16 x i32>, <16 x i32>* %q
%a = icmp eq <16 x i8> %s, %t
%b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y
store <16 x i32> %b, <16 x i32>* %r
ret void
}
define dso_local void @vselect_split_v16i16_setcc(<16 x i16> %s, <16 x i16> %t, <16 x i32>* %p, <16 x i32>* %q, <16 x i32>* %r) "min-legal-vector-width"="256" {
; CHECK-LABEL: vselect_split_v16i16_setcc:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rsi), %ymm2
; CHECK-NEXT: vmovdqa 32(%rsi), %ymm3
; CHECK-NEXT: vpcmpeqw %ymm1, %ymm0, %k1
; CHECK-NEXT: kshiftrw $8, %k1, %k2
; CHECK-NEXT: vmovdqa32 32(%rdi), %ymm3 {%k2}
; CHECK-NEXT: vmovdqa32 (%rdi), %ymm2 {%k1}
; CHECK-NEXT: vmovdqa %ymm2, (%rdx)
; CHECK-NEXT: vmovdqa %ymm3, 32(%rdx)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%x = load <16 x i32>, <16 x i32>* %p
%y = load <16 x i32>, <16 x i32>* %q
%a = icmp eq <16 x i16> %s, %t
%b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y
store <16 x i32> %b, <16 x i32>* %r
ret void
}
define <16 x i8> @trunc_packus_v16i32_v16i8(<16 x i32>* %p) "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_packus_v16i32_v16i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rdi), %ymm0
; CHECK-NEXT: vpackusdw 32(%rdi), %ymm0, %ymm0
; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-NEXT: vpmovuswb %ymm0, %xmm0
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%a = load <16 x i32>, <16 x i32>* %p
%b = icmp slt <16 x i32> %a, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
%c = select <16 x i1> %b, <16 x i32> %a, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
%d = icmp sgt <16 x i32> %c, zeroinitializer
%e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> zeroinitializer
%f = trunc <16 x i32> %e to <16 x i8>
ret <16 x i8> %f
}
define dso_local void @trunc_packus_v16i32_v16i8_store(<16 x i32>* %p, <16 x i8>* %q) "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_packus_v16i32_v16i8_store:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rdi), %ymm0
; CHECK-NEXT: vpackusdw 32(%rdi), %ymm0, %ymm0
; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-NEXT: vpmovuswb %ymm0, (%rsi)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%a = load <16 x i32>, <16 x i32>* %p
%b = icmp slt <16 x i32> %a, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
%c = select <16 x i1> %b, <16 x i32> %a, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
%d = icmp sgt <16 x i32> %c, zeroinitializer
%e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> zeroinitializer
%f = trunc <16 x i32> %e to <16 x i8>
store <16 x i8> %f, <16 x i8>* %q
ret void
}
define <64 x i1> @v64i1_argument_return(<64 x i1> %x) "min-legal-vector-width"="256" {
; CHECK-LABEL: v64i1_argument_return:
; CHECK: # %bb.0:
; CHECK-NEXT: retq
ret <64 x i1> %x
}
define dso_local void @v64i1_shuffle(<64 x i8>* %x, <64 x i8>* %y) "min-legal-vector-width"="256" {
; CHECK-LABEL: v64i1_shuffle:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vmovdqa (%rdi), %ymm1
; CHECK-NEXT: vmovdqa 32(%rdi), %ymm0
; CHECK-NEXT: vptestnmb %ymm1, %ymm1, %k0
; CHECK-NEXT: kshiftrd $1, %k0, %k1
; CHECK-NEXT: kshiftlq $63, %k0, %k2
; CHECK-NEXT: kshiftrq $62, %k2, %k2
; CHECK-NEXT: kshiftlq $63, %k1, %k1
; CHECK-NEXT: kshiftrq $63, %k1, %k1
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-5, %rax
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $3, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $61, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-9, %rax
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $2, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $60, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-17, %rax
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $5, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $59, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-33, %rax
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $4, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $58, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-65, %rax
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $7, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $57, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-129, %rax
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $6, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $56, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-257, %rax # imm = 0xFEFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $9, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $55, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-513, %rax # imm = 0xFDFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $8, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $54, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-1025, %rax # imm = 0xFBFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $11, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $53, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-2049, %rax # imm = 0xF7FF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $10, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $52, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-4097, %rax # imm = 0xEFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $13, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $51, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-8193, %rax # imm = 0xDFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $12, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $50, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-16385, %rax # imm = 0xBFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $15, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $49, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-32769, %rax # imm = 0xFFFF7FFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $14, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $48, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-65537, %rax # imm = 0xFFFEFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $17, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $47, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-131073, %rax # imm = 0xFFFDFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $16, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $46, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-262145, %rax # imm = 0xFFFBFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $19, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $45, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-524289, %rax # imm = 0xFFF7FFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $18, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $44, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-1048577, %rax # imm = 0xFFEFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $21, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $43, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-2097153, %rax # imm = 0xFFDFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $20, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $42, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-4194305, %rax # imm = 0xFFBFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $23, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $41, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-8388609, %rax # imm = 0xFF7FFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $22, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $40, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-16777217, %rax # imm = 0xFEFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $25, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $39, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-33554433, %rax # imm = 0xFDFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $24, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $38, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-67108865, %rax # imm = 0xFBFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $27, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $37, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-134217729, %rax # imm = 0xF7FFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $26, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $36, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-268435457, %rax # imm = 0xEFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $29, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $35, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-536870913, %rax # imm = 0xDFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $28, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $34, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movq $-1073741825, %rax # imm = 0xBFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k1
; CHECK-NEXT: kshiftrd $31, %k0, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $33, %k2, %k2
; CHECK-NEXT: korq %k2, %k1, %k1
; CHECK-NEXT: movabsq $-2147483649, %rax # imm = 0xFFFFFFFF7FFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k1, %k2
; CHECK-NEXT: vptestnmb %ymm0, %ymm0, %k1
; CHECK-NEXT: kshiftrd $30, %k0, %k0
; CHECK-NEXT: kshiftlq $63, %k0, %k0
; CHECK-NEXT: kshiftrq $32, %k0, %k0
; CHECK-NEXT: korq %k0, %k2, %k0
; CHECK-NEXT: movabsq $-4294967297, %rax # imm = 0xFFFFFFFEFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $1, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $31, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-8589934593, %rax # imm = 0xFFFFFFFDFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftlq $63, %k1, %k2
; CHECK-NEXT: kshiftrq $30, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-17179869185, %rax # imm = 0xFFFFFFFBFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $3, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $29, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-34359738369, %rax # imm = 0xFFFFFFF7FFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $2, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $28, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-68719476737, %rax # imm = 0xFFFFFFEFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $5, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $27, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-137438953473, %rax # imm = 0xFFFFFFDFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $4, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $26, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-274877906945, %rax # imm = 0xFFFFFFBFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $7, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $25, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-549755813889, %rax # imm = 0xFFFFFF7FFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $6, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $24, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-1099511627777, %rax # imm = 0xFFFFFEFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $9, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $23, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-2199023255553, %rax # imm = 0xFFFFFDFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $8, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $22, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-4398046511105, %rax # imm = 0xFFFFFBFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $11, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $21, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-8796093022209, %rax # imm = 0xFFFFF7FFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $10, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $20, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-17592186044417, %rax # imm = 0xFFFFEFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $13, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $19, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-35184372088833, %rax # imm = 0xFFFFDFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $12, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $18, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-70368744177665, %rax # imm = 0xFFFFBFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $15, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $17, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-140737488355329, %rax # imm = 0xFFFF7FFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $14, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $16, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-281474976710657, %rax # imm = 0xFFFEFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $17, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $15, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-562949953421313, %rax # imm = 0xFFFDFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $16, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $14, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-1125899906842625, %rax # imm = 0xFFFBFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $19, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $13, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-2251799813685249, %rax # imm = 0xFFF7FFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $18, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $12, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-4503599627370497, %rax # imm = 0xFFEFFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $21, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $11, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-9007199254740993, %rax # imm = 0xFFDFFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $20, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $10, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-18014398509481985, %rax # imm = 0xFFBFFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $23, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $9, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-36028797018963969, %rax # imm = 0xFF7FFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $22, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $8, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-72057594037927937, %rax # imm = 0xFEFFFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $25, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $7, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-144115188075855873, %rax # imm = 0xFDFFFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $24, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $6, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-288230376151711745, %rax # imm = 0xFBFFFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $27, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $5, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-576460752303423489, %rax # imm = 0xF7FFFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $26, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $4, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-1152921504606846977, %rax # imm = 0xEFFFFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $29, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $3, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-2305843009213693953, %rax # imm = 0xDFFFFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $28, %k1, %k2
; CHECK-NEXT: kshiftlq $63, %k2, %k2
; CHECK-NEXT: kshiftrq $2, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: movabsq $-4611686018427387905, %rax # imm = 0xBFFFFFFFFFFFFFFF
; CHECK-NEXT: kmovq %rax, %k2
; CHECK-NEXT: kandq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $31, %k1, %k2
; CHECK-NEXT: kshiftlq $62, %k2, %k2
; CHECK-NEXT: korq %k2, %k0, %k0
; CHECK-NEXT: kshiftrd $30, %k1, %k1
; CHECK-NEXT: kshiftlq $1, %k0, %k0
; CHECK-NEXT: kshiftrq $1, %k0, %k0
; CHECK-NEXT: kshiftlq $63, %k1, %k1
; CHECK-NEXT: korq %k1, %k0, %k1
; CHECK-NEXT: vmovdqu8 %ymm1, (%rsi) {%k1}
; CHECK-NEXT: kshiftrq $32, %k1, %k1
; CHECK-NEXT: vmovdqu8 %ymm0, 32(%rsi) {%k1}
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
entry:
%a = load <64 x i8>, <64 x i8>* %x
%b = icmp eq <64 x i8> %a, zeroinitializer
%shuf = shufflevector <64 x i1> %b, <64 x i1> undef, <64 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30, i32 33, i32 32, i32 35, i32 34, i32 37, i32 36, i32 39, i32 38, i32 41, i32 40, i32 43, i32 42, i32 45, i32 44, i32 47, i32 46, i32 49, i32 48, i32 51, i32 50, i32 53, i32 52, i32 55, i32 54, i32 57, i32 56, i32 59, i32 58, i32 61, i32 60, i32 63, i32 62>
call void @llvm.masked.store.v64i8.p0v64i8(<64 x i8> %a, <64 x i8>* %y, i32 1, <64 x i1> %shuf)
ret void
}
declare void @llvm.masked.store.v64i8.p0v64i8(<64 x i8>, <64 x i8>*, i32, <64 x i1>)
@mem64_dst = dso_local global i64 0, align 8
@mem64_src = dso_local global i64 0, align 8
define dso_local i32 @v64i1_inline_asm() "min-legal-vector-width"="256" {
; CHECK-LABEL: v64i1_inline_asm:
; CHECK: # %bb.0:
; CHECK-NEXT: kmovq mem64_src(%rip), %k0
; CHECK-NEXT: #APP
; CHECK-NEXT: #NO_APP
; CHECK-NEXT: kmovq %k0, mem64_dst(%rip)
; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %eax
; CHECK-NEXT: retq
%1 = alloca i32, align 4
%2 = load i64, i64* @mem64_src, align 8
%3 = call i64 asm "", "=k,k,~{dirflag},~{fpsr},~{flags}"(i64 %2)
store i64 %3, i64* @mem64_dst, align 8
%4 = load i32, i32* %1, align 4
ret i32 %4
}
define dso_local void @cmp_v8i64_sext(<8 x i64>* %xptr, <8 x i64>* %yptr, <8 x i64>* %zptr) "min-legal-vector-width"="256" {
; CHECK-LABEL: cmp_v8i64_sext:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rsi), %ymm0
; CHECK-NEXT: vmovdqa 32(%rsi), %ymm1
; CHECK-NEXT: vpcmpgtq 32(%rdi), %ymm1, %ymm1
; CHECK-NEXT: vpcmpgtq (%rdi), %ymm0, %ymm0
; CHECK-NEXT: vmovdqa %ymm0, (%rdx)
; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%x = load <8 x i64>, <8 x i64>* %xptr
%y = load <8 x i64>, <8 x i64>* %yptr
%cmp = icmp slt <8 x i64> %x, %y
%ext = sext <8 x i1> %cmp to <8 x i64>
store <8 x i64> %ext, <8 x i64>* %zptr
ret void
}
define dso_local void @cmp_v8i64_zext(<8 x i64>* %xptr, <8 x i64>* %yptr, <8 x i64>* %zptr) "min-legal-vector-width"="256" {
; CHECK-LABEL: cmp_v8i64_zext:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovdqa (%rsi), %ymm0
; CHECK-NEXT: vmovdqa 32(%rsi), %ymm1
; CHECK-NEXT: vpcmpgtq 32(%rdi), %ymm1, %ymm1
; CHECK-NEXT: vpcmpgtq (%rdi), %ymm0, %ymm0
; CHECK-NEXT: vpsrlq $63, %ymm1, %ymm1
; CHECK-NEXT: vpsrlq $63, %ymm0, %ymm0
; CHECK-NEXT: vmovdqa %ymm0, (%rdx)
; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT: vzeroupper
; CHECK-NEXT: retq
%x = load <8 x i64>, <8 x i64>* %xptr
%y = load <8 x i64>, <8 x i64>* %yptr
%cmp = icmp slt <8 x i64> %x, %y
%ext = zext <8 x i1> %cmp to <8 x i64>
store <8 x i64> %ext, <8 x i64>* %zptr
ret void
}
define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: var_rotate_v16i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm1
; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2
; CHECK-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
; CHECK-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; CHECK-NEXT: vpsllvw %xmm2, %xmm3, %xmm2
; CHECK-NEXT: vpsrlw $8, %xmm2, %xmm2
; CHECK-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
; CHECK-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; CHECK-NEXT: vpsllvw %xmm1, %xmm0, %xmm0
; CHECK-NEXT: vpsrlw $8, %xmm0, %xmm0
; CHECK-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
; CHECK-NEXT: retq
%b8 = sub <16 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %b
%shl = shl <16 x i8> %a, %b
%lshr = lshr <16 x i8> %a, %b8
%or = or <16 x i8> %shl, %lshr
ret <16 x i8> %or
}
define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: var_rotate_v32i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2
; CHECK-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
; CHECK-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-NEXT: vpsllvw %ymm3, %ymm4, %ymm3
; CHECK-NEXT: vpsrlw $8, %ymm3, %ymm3
; CHECK-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
; CHECK-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-NEXT: vpsllvw %ymm1, %ymm0, %ymm0
; CHECK-NEXT: vpsrlw $8, %ymm0, %ymm0
; CHECK-NEXT: vpackuswb %ymm3, %ymm0, %ymm0
; CHECK-NEXT: retq
%b8 = sub <32 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %b
%shl = shl <32 x i8> %a, %b
%lshr = lshr <32 x i8> %a, %b8
%or = or <32 x i8> %shl, %lshr
ret <32 x i8> %or
}
define <32 x i8> @splatvar_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: splatvar_rotate_v32i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
; CHECK-NEXT: vpsllw %xmm1, %ymm2, %ymm2
; CHECK-NEXT: vpsrlw $8, %ymm2, %ymm2
; CHECK-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-NEXT: vpsllw %xmm1, %ymm0, %ymm0
; CHECK-NEXT: vpsrlw $8, %ymm0, %ymm0
; CHECK-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
; CHECK-NEXT: retq
%splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
%splat8 = sub <32 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %splat
%shl = shl <32 x i8> %a, %splat
%lshr = lshr <32 x i8> %a, %splat8
%or = or <32 x i8> %shl, %lshr
ret <32 x i8> %or
}
define <32 x i8> @constant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: constant_rotate_v32i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; CHECK-NEXT: vpsrlw $8, %ymm1, %ymm1
; CHECK-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; CHECK-NEXT: vpsrlw $8, %ymm0, %ymm0
; CHECK-NEXT: vpackuswb %ymm1, %ymm0, %ymm0
; CHECK-NEXT: retq
%shl = shl <32 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>
%lshr = lshr <32 x i8> %a, <i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
%or = or <32 x i8> %shl, %lshr
ret <32 x i8> %or
}
define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: splatconstant_rotate_v32i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vpsllw $4, %ymm0, %ymm1
; CHECK-NEXT: vpsrlw $4, %ymm0, %ymm0
; CHECK-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0
; CHECK-NEXT: retq
%shl = shl <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
%lshr = lshr <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
%or = or <32 x i8> %shl, %lshr
ret <32 x i8> %or
}
define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: splatconstant_rotate_mask_v32i8:
; CHECK: # %bb.0:
; CHECK-NEXT: vpsllw $4, %ymm0, %ymm1
; CHECK-NEXT: vpsrlw $4, %ymm0, %ymm0
; CHECK-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0
; CHECK-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
; CHECK-NEXT: retq
%shl = shl <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
%lshr = lshr <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
%rmask = and <32 x i8> %lshr, <i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55>
%lmask = and <32 x i8> %shl, <i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33>
%or = or <32 x i8> %lmask, %rmask
ret <32 x i8> %or
}