; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -mattr=prefer-256-bit | FileCheck %s --check-prefixes=CHECK,CHECK-SKX,CHECK-SKX-NOVBMI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -mattr=prefer-256-bit,avx512vbmi | FileCheck %s --check-prefixes=CHECK,CHECK-SKX,CHECK-SKX-VBMI
; Make sure CPUs default to prefer-256-bit. avx512vnni isn't interesting as it just adds an isel peephole for vpmaddwd+vpaddd
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cascadelake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cooperlake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=cannonlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI,CHECK-VBMI1
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-client | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI,CHECK-GFNI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-server | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI,CHECK-GFNI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI,CHECK-GFNI

; This file primarily contains tests for specific places in X86ISelLowering.cpp that needed be made aware of the legalizer not allowing 512-bit vectors due to prefer-256-bit even though AVX512 is enabled.

define dso_local void @add256(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="256" {
; CHECK-LABEL: add256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vpaddd 32(%rsi), %ymm1, %ymm1
; CHECK-NEXT:    vpaddd (%rsi), %ymm0, %ymm0
; CHECK-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %d = load <16 x i32>, ptr %a
  %e = load <16 x i32>, ptr %b
  %f = add <16 x i32> %d, %e
  store <16 x i32> %f, ptr %c
  ret void
}

define dso_local void @add512(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="512" {
; CHECK-LABEL: add512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
; CHECK-NEXT:    vpaddd (%rsi), %zmm0, %zmm0
; CHECK-NEXT:    vmovdqa64 %zmm0, (%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %d = load <16 x i32>, ptr %a
  %e = load <16 x i32>, ptr %b
  %f = add <16 x i32> %d, %e
  store <16 x i32> %f, ptr %c
  ret void
}

define dso_local void @avg_v64i8_256(ptr %a, ptr %b) "min-legal-vector-width"="256" {
; CHECK-LABEL: avg_v64i8_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vpavgb (%rsi), %ymm0, %ymm0
; CHECK-NEXT:    vpavgb 32(%rsi), %ymm1, %ymm1
; CHECK-NEXT:    vmovdqu %ymm1, (%rax)
; CHECK-NEXT:    vmovdqu %ymm0, (%rax)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %1 = load <64 x i8>, ptr %a
  %2 = load <64 x i8>, ptr %b
  %3 = zext <64 x i8> %1 to <64 x i32>
  %4 = zext <64 x i8> %2 to <64 x i32>
  %5 = add nuw nsw <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %6 = add nuw nsw <64 x i32> %5, %4
  %7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %8 = trunc <64 x i32> %7 to <64 x i8>
  store <64 x i8> %8, ptr undef, align 4
  ret void
}


define dso_local void @avg_v64i8_512(ptr %a, ptr %b) "min-legal-vector-width"="512" {
; CHECK-LABEL: avg_v64i8_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
; CHECK-NEXT:    vpavgb (%rsi), %zmm0, %zmm0
; CHECK-NEXT:    vmovdqu64 %zmm0, (%rax)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %1 = load <64 x i8>, ptr %a
  %2 = load <64 x i8>, ptr %b
  %3 = zext <64 x i8> %1 to <64 x i32>
  %4 = zext <64 x i8> %2 to <64 x i32>
  %5 = add nuw nsw <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %6 = add nuw nsw <64 x i32> %5, %4
  %7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %8 = trunc <64 x i32> %7 to <64 x i8>
  store <64 x i8> %8, ptr undef, align 4
  ret void
}

define dso_local void @pmaddwd_32_256(ptr %APtr, ptr %BPtr, ptr %CPtr) "min-legal-vector-width"="256" {
; CHECK-LABEL: pmaddwd_32_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vpmaddwd 32(%rsi), %ymm1, %ymm1
; CHECK-NEXT:    vpmaddwd (%rsi), %ymm0, %ymm0
; CHECK-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
   %A = load <32 x i16>, ptr %APtr
   %B = load <32 x i16>, ptr %BPtr
   %a = sext <32 x i16> %A to <32 x i32>
   %b = sext <32 x i16> %B to <32 x i32>
   %m = mul nsw <32 x i32> %a, %b
   %odd = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
   %even = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
   %ret = add <16 x i32> %odd, %even
   store <16 x i32> %ret, ptr %CPtr
   ret void
}

define dso_local void @pmaddwd_32_512(ptr %APtr, ptr %BPtr, ptr %CPtr) "min-legal-vector-width"="512" {
; CHECK-LABEL: pmaddwd_32_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
; CHECK-NEXT:    vpmaddwd (%rsi), %zmm0, %zmm0
; CHECK-NEXT:    vmovdqa64 %zmm0, (%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
   %A = load <32 x i16>, ptr %APtr
   %B = load <32 x i16>, ptr %BPtr
   %a = sext <32 x i16> %A to <32 x i32>
   %b = sext <32 x i16> %B to <32 x i32>
   %m = mul nsw <32 x i32> %a, %b
   %odd = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
   %even = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
   %ret = add <16 x i32> %odd, %even
   store <16 x i32> %ret, ptr %CPtr
   ret void
}

define dso_local void @psubus_64i8_max_256(ptr %xptr, ptr %yptr, ptr %zptr) "min-legal-vector-width"="256" {
; CHECK-LABEL: psubus_64i8_max_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vpsubusb 32(%rsi), %ymm1, %ymm1
; CHECK-NEXT:    vpsubusb (%rsi), %ymm0, %ymm0
; CHECK-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <64 x i8>, ptr %xptr
  %y = load <64 x i8>, ptr %yptr
  %cmp = icmp ult <64 x i8> %x, %y
  %max = select <64 x i1> %cmp, <64 x i8> %y, <64 x i8> %x
  %res = sub <64 x i8> %max, %y
  store <64 x i8> %res, ptr %zptr
  ret void
}

define dso_local void @psubus_64i8_max_512(ptr %xptr, ptr %yptr, ptr %zptr) "min-legal-vector-width"="512" {
; CHECK-LABEL: psubus_64i8_max_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
; CHECK-NEXT:    vpsubusb (%rsi), %zmm0, %zmm0
; CHECK-NEXT:    vmovdqa64 %zmm0, (%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <64 x i8>, ptr %xptr
  %y = load <64 x i8>, ptr %yptr
  %cmp = icmp ult <64 x i8> %x, %y
  %max = select <64 x i1> %cmp, <64 x i8> %y, <64 x i8> %x
  %res = sub <64 x i8> %max, %y
  store <64 x i8> %res, ptr %zptr
  ret void
}

define dso_local i32 @_Z9test_charPcS_i_256(ptr nocapture readonly, ptr nocapture readonly, i32) "min-legal-vector-width"="256" {
; CHECK-SKX-LABEL: _Z9test_charPcS_i_256:
; CHECK-SKX:       # %bb.0: # %entry
; CHECK-SKX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-SKX-NEXT:    movl %edx, %eax
; CHECK-SKX-NEXT:    xorl %ecx, %ecx
; CHECK-SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; CHECK-SKX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
; CHECK-SKX-NEXT:    .p2align 4
; CHECK-SKX-NEXT:  .LBB8_1: # %vector.body
; CHECK-SKX-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-SKX-NEXT:    vpmovsxbw 16(%rdi,%rcx), %ymm3
; CHECK-SKX-NEXT:    vpmovsxbw (%rdi,%rcx), %ymm4
; CHECK-SKX-NEXT:    vpmovsxbw 16(%rsi,%rcx), %ymm5
; CHECK-SKX-NEXT:    vpmaddwd %ymm3, %ymm5, %ymm3
; CHECK-SKX-NEXT:    vpaddd %ymm2, %ymm3, %ymm2
; CHECK-SKX-NEXT:    vpmovsxbw (%rsi,%rcx), %ymm3
; CHECK-SKX-NEXT:    vpmaddwd %ymm4, %ymm3, %ymm3
; CHECK-SKX-NEXT:    vpaddd %ymm1, %ymm3, %ymm1
; CHECK-SKX-NEXT:    addq $32, %rcx
; CHECK-SKX-NEXT:    cmpq %rcx, %rax
; CHECK-SKX-NEXT:    jne .LBB8_1
; CHECK-SKX-NEXT:  # %bb.2: # %middle.block
; CHECK-SKX-NEXT:    vpaddd %ymm0, %ymm1, %ymm1
; CHECK-SKX-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
; CHECK-SKX-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
; CHECK-SKX-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT:    vpsrlq $32, %xmm0, %xmm1
; CHECK-SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT:    vmovd %xmm0, %eax
; CHECK-SKX-NEXT:    vzeroupper
; CHECK-SKX-NEXT:    retq
;
; CHECK-AVX512-LABEL: _Z9test_charPcS_i_256:
; CHECK-AVX512:       # %bb.0: # %entry
; CHECK-AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    movl %edx, %eax
; CHECK-AVX512-NEXT:    xorl %ecx, %ecx
; CHECK-AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; CHECK-AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
; CHECK-AVX512-NEXT:    .p2align 4
; CHECK-AVX512-NEXT:  .LBB8_1: # %vector.body
; CHECK-AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-AVX512-NEXT:    vpmovsxbw 16(%rdi,%rcx), %ymm3
; CHECK-AVX512-NEXT:    vpmovsxbw (%rdi,%rcx), %ymm4
; CHECK-AVX512-NEXT:    vpmovsxbw 16(%rsi,%rcx), %ymm5
; CHECK-AVX512-NEXT:    vpmaddwd %ymm3, %ymm5, %ymm3
; CHECK-AVX512-NEXT:    vpaddd %ymm2, %ymm3, %ymm2
; CHECK-AVX512-NEXT:    vpmovsxbw (%rsi,%rcx), %ymm3
; CHECK-AVX512-NEXT:    vpmaddwd %ymm4, %ymm3, %ymm3
; CHECK-AVX512-NEXT:    vpaddd %ymm1, %ymm3, %ymm1
; CHECK-AVX512-NEXT:    addq $32, %rcx
; CHECK-AVX512-NEXT:    cmpq %rcx, %rax
; CHECK-AVX512-NEXT:    jne .LBB8_1
; CHECK-AVX512-NEXT:  # %bb.2: # %middle.block
; CHECK-AVX512-NEXT:    vpaddd %ymm0, %ymm1, %ymm1
; CHECK-AVX512-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
; CHECK-AVX512-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
; CHECK-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    vpsrlq $32, %xmm0, %xmm1
; CHECK-AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    vmovd %xmm0, %eax
; CHECK-AVX512-NEXT:    vzeroupper
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: _Z9test_charPcS_i_256:
; CHECK-VBMI:       # %bb.0: # %entry
; CHECK-VBMI-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    movl %edx, %eax
; CHECK-VBMI-NEXT:    xorl %ecx, %ecx
; CHECK-VBMI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; CHECK-VBMI-NEXT:    vpxor %xmm2, %xmm2, %xmm2
; CHECK-VBMI-NEXT:    .p2align 4
; CHECK-VBMI-NEXT:  .LBB8_1: # %vector.body
; CHECK-VBMI-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-VBMI-NEXT:    vpmovsxbw 16(%rdi,%rcx), %ymm3
; CHECK-VBMI-NEXT:    vpmovsxbw (%rdi,%rcx), %ymm4
; CHECK-VBMI-NEXT:    vpmovsxbw 16(%rsi,%rcx), %ymm5
; CHECK-VBMI-NEXT:    vpmaddwd %ymm3, %ymm5, %ymm3
; CHECK-VBMI-NEXT:    vpaddd %ymm2, %ymm3, %ymm2
; CHECK-VBMI-NEXT:    vpmovsxbw (%rsi,%rcx), %ymm3
; CHECK-VBMI-NEXT:    vpmaddwd %ymm4, %ymm3, %ymm3
; CHECK-VBMI-NEXT:    vpaddd %ymm1, %ymm3, %ymm1
; CHECK-VBMI-NEXT:    addq $32, %rcx
; CHECK-VBMI-NEXT:    cmpq %rcx, %rax
; CHECK-VBMI-NEXT:    jne .LBB8_1
; CHECK-VBMI-NEXT:  # %bb.2: # %middle.block
; CHECK-VBMI-NEXT:    vpaddd %ymm0, %ymm1, %ymm1
; CHECK-VBMI-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
; CHECK-VBMI-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
; CHECK-VBMI-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-VBMI-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-VBMI-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; CHECK-VBMI-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    vmovd %xmm0, %eax
; CHECK-VBMI-NEXT:    vzeroupper
; CHECK-VBMI-NEXT:    retq
entry:
  %3 = zext i32 %2 to i64
  br label %vector.body

vector.body:
  %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
  %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
  %4 = getelementptr inbounds i8, ptr %0, i64 %index
  %5 = bitcast ptr %4 to ptr
  %wide.load = load <32 x i8>, ptr %5, align 1
  %6 = sext <32 x i8> %wide.load to <32 x i32>
  %7 = getelementptr inbounds i8, ptr %1, i64 %index
  %8 = bitcast ptr %7 to ptr
  %wide.load14 = load <32 x i8>, ptr %8, align 1
  %9 = sext <32 x i8> %wide.load14 to <32 x i32>
  %10 = mul nsw <32 x i32> %9, %6
  %11 = add nsw <32 x i32> %10, %vec.phi
  %index.next = add i64 %index, 32
  %12 = icmp eq i64 %index.next, %3
  br i1 %12, label %middle.block, label %vector.body

middle.block:
  %13 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %11)
  ret i32 %13
}

define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocapture readonly, i32) "min-legal-vector-width"="512" {
; CHECK-SKX-LABEL: _Z9test_charPcS_i_512:
; CHECK-SKX:       # %bb.0: # %entry
; CHECK-SKX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-SKX-NEXT:    movl %edx, %eax
; CHECK-SKX-NEXT:    xorl %ecx, %ecx
; CHECK-SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; CHECK-SKX-NEXT:    .p2align 4
; CHECK-SKX-NEXT:  .LBB9_1: # %vector.body
; CHECK-SKX-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-SKX-NEXT:    vpmovsxbw (%rdi,%rcx), %zmm2
; CHECK-SKX-NEXT:    vpmovsxbw (%rsi,%rcx), %zmm3
; CHECK-SKX-NEXT:    vpmaddwd %zmm2, %zmm3, %zmm2
; CHECK-SKX-NEXT:    vpaddd %zmm1, %zmm2, %zmm1
; CHECK-SKX-NEXT:    addq $32, %rcx
; CHECK-SKX-NEXT:    cmpq %rcx, %rax
; CHECK-SKX-NEXT:    jne .LBB9_1
; CHECK-SKX-NEXT:  # %bb.2: # %middle.block
; CHECK-SKX-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
; CHECK-SKX-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
; CHECK-SKX-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
; CHECK-SKX-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT:    vpsrlq $32, %xmm0, %xmm1
; CHECK-SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT:    vmovd %xmm0, %eax
; CHECK-SKX-NEXT:    vzeroupper
; CHECK-SKX-NEXT:    retq
;
; CHECK-AVX512-LABEL: _Z9test_charPcS_i_512:
; CHECK-AVX512:       # %bb.0: # %entry
; CHECK-AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    movl %edx, %eax
; CHECK-AVX512-NEXT:    xorl %ecx, %ecx
; CHECK-AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; CHECK-AVX512-NEXT:    .p2align 4
; CHECK-AVX512-NEXT:  .LBB9_1: # %vector.body
; CHECK-AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-AVX512-NEXT:    vpmovsxbw (%rdi,%rcx), %zmm2
; CHECK-AVX512-NEXT:    vpmovsxbw (%rsi,%rcx), %zmm3
; CHECK-AVX512-NEXT:    vpmaddwd %zmm2, %zmm3, %zmm2
; CHECK-AVX512-NEXT:    vpaddd %zmm1, %zmm2, %zmm1
; CHECK-AVX512-NEXT:    addq $32, %rcx
; CHECK-AVX512-NEXT:    cmpq %rcx, %rax
; CHECK-AVX512-NEXT:    jne .LBB9_1
; CHECK-AVX512-NEXT:  # %bb.2: # %middle.block
; CHECK-AVX512-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
; CHECK-AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
; CHECK-AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
; CHECK-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    vpsrlq $32, %xmm0, %xmm1
; CHECK-AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    vmovd %xmm0, %eax
; CHECK-AVX512-NEXT:    vzeroupper
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: _Z9test_charPcS_i_512:
; CHECK-VBMI:       # %bb.0: # %entry
; CHECK-VBMI-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    movl %edx, %eax
; CHECK-VBMI-NEXT:    xorl %ecx, %ecx
; CHECK-VBMI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; CHECK-VBMI-NEXT:    .p2align 4
; CHECK-VBMI-NEXT:  .LBB9_1: # %vector.body
; CHECK-VBMI-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-VBMI-NEXT:    vpmovsxbw (%rdi,%rcx), %zmm2
; CHECK-VBMI-NEXT:    vpmovsxbw (%rsi,%rcx), %zmm3
; CHECK-VBMI-NEXT:    vpmaddwd %zmm2, %zmm3, %zmm2
; CHECK-VBMI-NEXT:    vpaddd %zmm1, %zmm2, %zmm1
; CHECK-VBMI-NEXT:    addq $32, %rcx
; CHECK-VBMI-NEXT:    cmpq %rcx, %rax
; CHECK-VBMI-NEXT:    jne .LBB9_1
; CHECK-VBMI-NEXT:  # %bb.2: # %middle.block
; CHECK-VBMI-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
; CHECK-VBMI-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
; CHECK-VBMI-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
; CHECK-VBMI-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-VBMI-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-VBMI-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; CHECK-VBMI-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    vmovd %xmm0, %eax
; CHECK-VBMI-NEXT:    vzeroupper
; CHECK-VBMI-NEXT:    retq
entry:
  %3 = zext i32 %2 to i64
  br label %vector.body

vector.body:
  %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
  %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
  %4 = getelementptr inbounds i8, ptr %0, i64 %index
  %5 = bitcast ptr %4 to ptr
  %wide.load = load <32 x i8>, ptr %5, align 1
  %6 = sext <32 x i8> %wide.load to <32 x i32>
  %7 = getelementptr inbounds i8, ptr %1, i64 %index
  %8 = bitcast ptr %7 to ptr
  %wide.load14 = load <32 x i8>, ptr %8, align 1
  %9 = sext <32 x i8> %wide.load14 to <32 x i32>
  %10 = mul nsw <32 x i32> %9, %6
  %11 = add nsw <32 x i32> %10, %vec.phi
  %index.next = add i64 %index, 32
  %12 = icmp eq i64 %index.next, %3
  br i1 %12, label %middle.block, label %vector.body

middle.block:
  %13 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %11)
  ret i32 %13
}

@a = dso_local global [1024 x i8] zeroinitializer, align 16
@b = dso_local global [1024 x i8] zeroinitializer, align 16

define dso_local i32 @sad_16i8_256() "min-legal-vector-width"="256" {
; CHECK-SKX-LABEL: sad_16i8_256:
; CHECK-SKX:       # %bb.0: # %entry
; CHECK-SKX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-SKX-NEXT:    movq $-1024, %rax # imm = 0xFC00
; CHECK-SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; CHECK-SKX-NEXT:    .p2align 4
; CHECK-SKX-NEXT:  .LBB10_1: # %vector.body
; CHECK-SKX-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-SKX-NEXT:    vmovdqu a+1024(%rax), %xmm2
; CHECK-SKX-NEXT:    vpsadbw b+1024(%rax), %xmm2, %xmm2
; CHECK-SKX-NEXT:    vpaddd %ymm1, %ymm2, %ymm1
; CHECK-SKX-NEXT:    addq $4, %rax
; CHECK-SKX-NEXT:    jne .LBB10_1
; CHECK-SKX-NEXT:  # %bb.2: # %middle.block
; CHECK-SKX-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
; CHECK-SKX-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT:    vpsrlq $32, %xmm0, %xmm1
; CHECK-SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT:    vmovd %xmm0, %eax
; CHECK-SKX-NEXT:    vzeroupper
; CHECK-SKX-NEXT:    retq
;
; CHECK-AVX512-LABEL: sad_16i8_256:
; CHECK-AVX512:       # %bb.0: # %entry
; CHECK-AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    movq $-1024, %rax # imm = 0xFC00
; CHECK-AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; CHECK-AVX512-NEXT:    .p2align 4
; CHECK-AVX512-NEXT:  .LBB10_1: # %vector.body
; CHECK-AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-AVX512-NEXT:    vmovdqu a+1024(%rax), %xmm2
; CHECK-AVX512-NEXT:    vpsadbw b+1024(%rax), %xmm2, %xmm2
; CHECK-AVX512-NEXT:    vpaddd %ymm1, %ymm2, %ymm1
; CHECK-AVX512-NEXT:    addq $4, %rax
; CHECK-AVX512-NEXT:    jne .LBB10_1
; CHECK-AVX512-NEXT:  # %bb.2: # %middle.block
; CHECK-AVX512-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
; CHECK-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    vpsrlq $32, %xmm0, %xmm1
; CHECK-AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    vmovd %xmm0, %eax
; CHECK-AVX512-NEXT:    vzeroupper
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: sad_16i8_256:
; CHECK-VBMI:       # %bb.0: # %entry
; CHECK-VBMI-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    movq $-1024, %rax # imm = 0xFC00
; CHECK-VBMI-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; CHECK-VBMI-NEXT:    .p2align 4
; CHECK-VBMI-NEXT:  .LBB10_1: # %vector.body
; CHECK-VBMI-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-VBMI-NEXT:    vmovdqu a+1024(%rax), %xmm2
; CHECK-VBMI-NEXT:    vpsadbw b+1024(%rax), %xmm2, %xmm2
; CHECK-VBMI-NEXT:    vpaddd %ymm1, %ymm2, %ymm1
; CHECK-VBMI-NEXT:    addq $4, %rax
; CHECK-VBMI-NEXT:    jne .LBB10_1
; CHECK-VBMI-NEXT:  # %bb.2: # %middle.block
; CHECK-VBMI-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
; CHECK-VBMI-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-VBMI-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-VBMI-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; CHECK-VBMI-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    vmovd %xmm0, %eax
; CHECK-VBMI-NEXT:    vzeroupper
; CHECK-VBMI-NEXT:    retq
entry:
  br label %vector.body

vector.body:
  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
  %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ]
  %0 = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
  %1 = bitcast ptr %0 to ptr
  %wide.load = load <16 x i8>, ptr %1, align 4
  %2 = zext <16 x i8> %wide.load to <16 x i32>
  %3 = getelementptr inbounds [1024 x i8], ptr @b, i64 0, i64 %index
  %4 = bitcast ptr %3 to ptr
  %wide.load1 = load <16 x i8>, ptr %4, align 4
  %5 = zext <16 x i8> %wide.load1 to <16 x i32>
  %6 = sub nsw <16 x i32> %2, %5
  %7 = icmp sgt <16 x i32> %6, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
  %8 = sub nsw <16 x i32> zeroinitializer, %6
  %9 = select <16 x i1> %7, <16 x i32> %6, <16 x i32> %8
  %10 = add nsw <16 x i32> %9, %vec.phi
  %index.next = add i64 %index, 4
  %11 = icmp eq i64 %index.next, 1024
  br i1 %11, label %middle.block, label %vector.body

middle.block:
  %12 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %10)
  ret i32 %12
}

define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" {
; CHECK-SKX-LABEL: sad_16i8_512:
; CHECK-SKX:       # %bb.0: # %entry
; CHECK-SKX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-SKX-NEXT:    movq $-1024, %rax # imm = 0xFC00
; CHECK-SKX-NEXT:    .p2align 4
; CHECK-SKX-NEXT:  .LBB11_1: # %vector.body
; CHECK-SKX-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-SKX-NEXT:    vmovdqu a+1024(%rax), %xmm1
; CHECK-SKX-NEXT:    vpsadbw b+1024(%rax), %xmm1, %xmm1
; CHECK-SKX-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
; CHECK-SKX-NEXT:    addq $4, %rax
; CHECK-SKX-NEXT:    jne .LBB11_1
; CHECK-SKX-NEXT:  # %bb.2: # %middle.block
; CHECK-SKX-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
; CHECK-SKX-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
; CHECK-SKX-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT:    vpsrlq $32, %xmm0, %xmm1
; CHECK-SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT:    vmovd %xmm0, %eax
; CHECK-SKX-NEXT:    vzeroupper
; CHECK-SKX-NEXT:    retq
;
; CHECK-AVX512-LABEL: sad_16i8_512:
; CHECK-AVX512:       # %bb.0: # %entry
; CHECK-AVX512-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    movq $-1024, %rax # imm = 0xFC00
; CHECK-AVX512-NEXT:    .p2align 4
; CHECK-AVX512-NEXT:  .LBB11_1: # %vector.body
; CHECK-AVX512-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-AVX512-NEXT:    vmovdqu a+1024(%rax), %xmm1
; CHECK-AVX512-NEXT:    vpsadbw b+1024(%rax), %xmm1, %xmm1
; CHECK-AVX512-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
; CHECK-AVX512-NEXT:    addq $4, %rax
; CHECK-AVX512-NEXT:    jne .LBB11_1
; CHECK-AVX512-NEXT:  # %bb.2: # %middle.block
; CHECK-AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
; CHECK-AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
; CHECK-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    vpsrlq $32, %xmm0, %xmm1
; CHECK-AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    vmovd %xmm0, %eax
; CHECK-AVX512-NEXT:    vzeroupper
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: sad_16i8_512:
; CHECK-VBMI:       # %bb.0: # %entry
; CHECK-VBMI-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    movq $-1024, %rax # imm = 0xFC00
; CHECK-VBMI-NEXT:    .p2align 4
; CHECK-VBMI-NEXT:  .LBB11_1: # %vector.body
; CHECK-VBMI-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-VBMI-NEXT:    vmovdqu a+1024(%rax), %xmm1
; CHECK-VBMI-NEXT:    vpsadbw b+1024(%rax), %xmm1, %xmm1
; CHECK-VBMI-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
; CHECK-VBMI-NEXT:    addq $4, %rax
; CHECK-VBMI-NEXT:    jne .LBB11_1
; CHECK-VBMI-NEXT:  # %bb.2: # %middle.block
; CHECK-VBMI-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
; CHECK-VBMI-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
; CHECK-VBMI-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-VBMI-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-VBMI-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; CHECK-VBMI-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT:    vmovd %xmm0, %eax
; CHECK-VBMI-NEXT:    vzeroupper
; CHECK-VBMI-NEXT:    retq
entry:
  br label %vector.body

vector.body:
  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
  %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ]
  %0 = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
  %1 = bitcast ptr %0 to ptr
  %wide.load = load <16 x i8>, ptr %1, align 4
  %2 = zext <16 x i8> %wide.load to <16 x i32>
  %3 = getelementptr inbounds [1024 x i8], ptr @b, i64 0, i64 %index
  %4 = bitcast ptr %3 to ptr
  %wide.load1 = load <16 x i8>, ptr %4, align 4
  %5 = zext <16 x i8> %wide.load1 to <16 x i32>
  %6 = sub nsw <16 x i32> %2, %5
  %7 = icmp sgt <16 x i32> %6, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
  %8 = sub nsw <16 x i32> zeroinitializer, %6
  %9 = select <16 x i1> %7, <16 x i32> %6, <16 x i32> %8
  %10 = add nsw <16 x i32> %9, %vec.phi
  %index.next = add i64 %index, 4
  %11 = icmp eq i64 %index.next, 1024
  br i1 %11, label %middle.block, label %vector.body

middle.block:
  %12 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %10)
  ret i32 %12
}

define dso_local void @sbto16f32_256(<16 x i16> %a, ptr %res) "min-legal-vector-width"="256" {
; CHECK-LABEL: sbto16f32_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    kshiftrw $8, %k0, %k1
; CHECK-NEXT:    vpmovm2d %k1, %ymm0
; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm0
; CHECK-NEXT:    vpmovm2d %k0, %ymm1
; CHECK-NEXT:    vcvtdq2ps %ymm1, %ymm1
; CHECK-NEXT:    vmovaps %ymm1, (%rdi)
; CHECK-NEXT:    vmovaps %ymm0, 32(%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = sitofp <16 x i1> %mask to <16 x float>
  store <16 x float> %1, ptr %res
  ret void
}

define dso_local void @sbto16f32_512(<16 x i16> %a, ptr %res) "min-legal-vector-width"="512" {
; CHECK-LABEL: sbto16f32_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    vpmovm2d %k0, %zmm0
; CHECK-NEXT:    vcvtdq2ps %zmm0, %zmm0
; CHECK-NEXT:    vmovaps %zmm0, (%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = sitofp <16 x i1> %mask to <16 x float>
  store <16 x float> %1, ptr %res
  ret void
}

define dso_local void @sbto16f64_256(<16 x i16> %a, ptr %res)  "min-legal-vector-width"="256" {
; CHECK-LABEL: sbto16f64_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    kshiftrw $8, %k0, %k1
; CHECK-NEXT:    vpmovm2d %k1, %ymm0
; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm1
; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm0
; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm0
; CHECK-NEXT:    vpmovm2d %k0, %ymm2
; CHECK-NEXT:    vcvtdq2pd %xmm2, %ymm3
; CHECK-NEXT:    vextracti128 $1, %ymm2, %xmm2
; CHECK-NEXT:    vcvtdq2pd %xmm2, %ymm2
; CHECK-NEXT:    vmovaps %ymm2, 32(%rdi)
; CHECK-NEXT:    vmovaps %ymm3, (%rdi)
; CHECK-NEXT:    vmovaps %ymm0, 96(%rdi)
; CHECK-NEXT:    vmovaps %ymm1, 64(%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = sitofp <16 x i1> %mask to <16 x double>
  store <16 x double> %1, ptr %res
  ret void
}

define dso_local void @sbto16f64_512(<16 x i16> %a, ptr %res)  "min-legal-vector-width"="512" {
; CHECK-LABEL: sbto16f64_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    vpmovm2d %k0, %zmm0
; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm1
; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm0
; CHECK-NEXT:    vmovaps %zmm0, 64(%rdi)
; CHECK-NEXT:    vmovaps %zmm1, (%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = sitofp <16 x i1> %mask to <16 x double>
  store <16 x double> %1, ptr %res
  ret void
}

define dso_local void @ubto16f32_256(<16 x i16> %a, ptr %res) "min-legal-vector-width"="256" {
; CHECK-LABEL: ubto16f32_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    kshiftrw $8, %k0, %k1
; CHECK-NEXT:    vpmovm2d %k1, %ymm0
; CHECK-NEXT:    vpsrld $31, %ymm0, %ymm0
; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm0
; CHECK-NEXT:    vpmovm2d %k0, %ymm1
; CHECK-NEXT:    vpsrld $31, %ymm1, %ymm1
; CHECK-NEXT:    vcvtdq2ps %ymm1, %ymm1
; CHECK-NEXT:    vmovaps %ymm1, (%rdi)
; CHECK-NEXT:    vmovaps %ymm0, 32(%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = uitofp <16 x i1> %mask to <16 x float>
  store <16 x float> %1, ptr %res
  ret void
}

define dso_local void @ubto16f32_512(<16 x i16> %a, ptr %res) "min-legal-vector-width"="512" {
; CHECK-LABEL: ubto16f32_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    vpmovm2d %k0, %zmm0
; CHECK-NEXT:    vpsrld $31, %zmm0, %zmm0
; CHECK-NEXT:    vcvtdq2ps %zmm0, %zmm0
; CHECK-NEXT:    vmovaps %zmm0, (%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = uitofp <16 x i1> %mask to <16 x float>
  store <16 x float> %1, ptr %res
  ret void
}

define dso_local void @ubto16f64_256(<16 x i16> %a, ptr %res) "min-legal-vector-width"="256" {
; CHECK-LABEL: ubto16f64_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    kshiftrw $8, %k0, %k1
; CHECK-NEXT:    vpmovm2d %k1, %ymm0
; CHECK-NEXT:    vpsrld $31, %ymm0, %ymm0
; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm1
; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm0
; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm0
; CHECK-NEXT:    vpmovm2d %k0, %ymm2
; CHECK-NEXT:    vpsrld $31, %ymm2, %ymm2
; CHECK-NEXT:    vcvtdq2pd %xmm2, %ymm3
; CHECK-NEXT:    vextracti128 $1, %ymm2, %xmm2
; CHECK-NEXT:    vcvtdq2pd %xmm2, %ymm2
; CHECK-NEXT:    vmovaps %ymm2, 32(%rdi)
; CHECK-NEXT:    vmovaps %ymm3, (%rdi)
; CHECK-NEXT:    vmovaps %ymm0, 96(%rdi)
; CHECK-NEXT:    vmovaps %ymm1, 64(%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = uitofp <16 x i1> %mask to <16 x double>
  store <16 x double> %1, ptr %res
  ret void
}

define dso_local void @ubto16f64_512(<16 x i16> %a, ptr %res) "min-legal-vector-width"="512" {
; CHECK-LABEL: ubto16f64_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    vpmovm2d %k0, %zmm0
; CHECK-NEXT:    vpsrld $31, %zmm0, %zmm0
; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm1
; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm0
; CHECK-NEXT:    vmovaps %zmm0, 64(%rdi)
; CHECK-NEXT:    vmovaps %zmm1, (%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = uitofp <16 x i1> %mask to <16 x double>
  store <16 x double> %1, ptr %res
  ret void
}

define <16 x i16> @test_16f32toub_256(ptr %ptr, <16 x i16> %passthru) "min-legal-vector-width"="256" {
; CHECK-LABEL: test_16f32toub_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vcvttps2dq (%rdi), %ymm1
; CHECK-NEXT:    vpslld $31, %ymm1, %ymm1
; CHECK-NEXT:    vpmovd2m %ymm1, %k0
; CHECK-NEXT:    vcvttps2dq 32(%rdi), %ymm1
; CHECK-NEXT:    vpslld $31, %ymm1, %ymm1
; CHECK-NEXT:    vpmovd2m %ymm1, %k1
; CHECK-NEXT:    kunpckbw %k0, %k1, %k1
; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm0 {%k1} {z}
; CHECK-NEXT:    retq
  %a = load <16 x float>, ptr %ptr
  %mask = fptoui <16 x float> %a to <16 x i1>
  %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer
  ret <16 x i16> %select
}

define <16 x i16> @test_16f32toub_512(ptr %ptr, <16 x i16> %passthru) "min-legal-vector-width"="512" {
; CHECK-LABEL: test_16f32toub_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vcvttps2dq (%rdi), %zmm1
; CHECK-NEXT:    vpslld $31, %zmm1, %zmm1
; CHECK-NEXT:    vpmovd2m %zmm1, %k1
; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm0 {%k1} {z}
; CHECK-NEXT:    retq
  %a = load <16 x float>, ptr %ptr
  %mask = fptoui <16 x float> %a to <16 x i1>
  %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer
  ret <16 x i16> %select
}

define <16 x i16> @test_16f32tosb_256(ptr %ptr, <16 x i16> %passthru) "min-legal-vector-width"="256" {
; CHECK-LABEL: test_16f32tosb_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vcvttps2dq (%rdi), %ymm1
; CHECK-NEXT:    vpmovd2m %ymm1, %k0
; CHECK-NEXT:    vcvttps2dq 32(%rdi), %ymm1
; CHECK-NEXT:    vpmovd2m %ymm1, %k1
; CHECK-NEXT:    kunpckbw %k0, %k1, %k1
; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm0 {%k1} {z}
; CHECK-NEXT:    retq
  %a = load <16 x float>, ptr %ptr
  %mask = fptosi <16 x float> %a to <16 x i1>
  %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer
  ret <16 x i16> %select
}

define <16 x i16> @test_16f32tosb_512(ptr %ptr, <16 x i16> %passthru) "min-legal-vector-width"="512" {
; CHECK-LABEL: test_16f32tosb_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vcvttps2dq (%rdi), %zmm1
; CHECK-NEXT:    vpmovd2m %zmm1, %k1
; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm0 {%k1} {z}
; CHECK-NEXT:    retq
  %a = load <16 x float>, ptr %ptr
  %mask = fptosi <16 x float> %a to <16 x i1>
  %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer
  ret <16 x i16> %select
}

define dso_local void @mul256(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="256" {
; CHECK-SKX-NOVBMI-LABEL: mul256:
; CHECK-SKX-NOVBMI:       # %bb.0:
; CHECK-SKX-NOVBMI-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-SKX-NOVBMI-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-SKX-NOVBMI-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-SKX-NOVBMI-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-SKX-NOVBMI-NEXT:    vpmullw %ymm3, %ymm1, %ymm4
; CHECK-SKX-NOVBMI-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; CHECK-SKX-NOVBMI-NEXT:    vpandn %ymm3, %ymm5, %ymm3
; CHECK-SKX-NOVBMI-NEXT:    vpmaddubsw %ymm3, %ymm1, %ymm1
; CHECK-SKX-NOVBMI-NEXT:    vpsllw $8, %ymm1, %ymm1
; CHECK-SKX-NOVBMI-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm5)
; CHECK-SKX-NOVBMI-NEXT:    vpmullw %ymm2, %ymm0, %ymm3
; CHECK-SKX-NOVBMI-NEXT:    vpandn %ymm2, %ymm5, %ymm2
; CHECK-SKX-NOVBMI-NEXT:    vpmaddubsw %ymm2, %ymm0, %ymm0
; CHECK-SKX-NOVBMI-NEXT:    vpsllw $8, %ymm0, %ymm0
; CHECK-SKX-NOVBMI-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm3 & ymm5)
; CHECK-SKX-NOVBMI-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-SKX-NOVBMI-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-SKX-NOVBMI-NEXT:    vzeroupper
; CHECK-SKX-NOVBMI-NEXT:    retq
;
; CHECK-SKX-VBMI-LABEL: mul256:
; CHECK-SKX-VBMI:       # %bb.0:
; CHECK-SKX-VBMI-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-SKX-VBMI-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-SKX-VBMI-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-SKX-VBMI-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-SKX-VBMI-NEXT:    vpmullw %ymm3, %ymm1, %ymm4
; CHECK-SKX-VBMI-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; CHECK-SKX-VBMI-NEXT:    vpandn %ymm3, %ymm5, %ymm3
; CHECK-SKX-VBMI-NEXT:    vpmaddubsw %ymm3, %ymm1, %ymm1
; CHECK-SKX-VBMI-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,32,2,34,4,36,6,38,8,40,10,42,12,44,14,46,16,48,18,50,20,52,22,54,24,56,26,58,28,60,30,62]
; CHECK-SKX-VBMI-NEXT:    vpermt2b %ymm1, %ymm3, %ymm4
; CHECK-SKX-VBMI-NEXT:    vpmullw %ymm2, %ymm0, %ymm1
; CHECK-SKX-VBMI-NEXT:    vpandn %ymm2, %ymm5, %ymm2
; CHECK-SKX-VBMI-NEXT:    vpmaddubsw %ymm2, %ymm0, %ymm0
; CHECK-SKX-VBMI-NEXT:    vpermt2b %ymm0, %ymm3, %ymm1
; CHECK-SKX-VBMI-NEXT:    vmovdqa %ymm1, (%rdx)
; CHECK-SKX-VBMI-NEXT:    vmovdqa %ymm4, 32(%rdx)
; CHECK-SKX-VBMI-NEXT:    vzeroupper
; CHECK-SKX-VBMI-NEXT:    retq
;
; CHECK-AVX512-LABEL: mul256:
; CHECK-AVX512:       # %bb.0:
; CHECK-AVX512-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-AVX512-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-AVX512-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-AVX512-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-AVX512-NEXT:    vpmullw %ymm3, %ymm1, %ymm4
; CHECK-AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; CHECK-AVX512-NEXT:    vpandn %ymm3, %ymm5, %ymm3
; CHECK-AVX512-NEXT:    vpmaddubsw %ymm3, %ymm1, %ymm1
; CHECK-AVX512-NEXT:    vpsllw $8, %ymm1, %ymm1
; CHECK-AVX512-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm5)
; CHECK-AVX512-NEXT:    vpmullw %ymm2, %ymm0, %ymm3
; CHECK-AVX512-NEXT:    vpandn %ymm2, %ymm5, %ymm2
; CHECK-AVX512-NEXT:    vpmaddubsw %ymm2, %ymm0, %ymm0
; CHECK-AVX512-NEXT:    vpsllw $8, %ymm0, %ymm0
; CHECK-AVX512-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm3 & ymm5)
; CHECK-AVX512-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-AVX512-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-AVX512-NEXT:    vzeroupper
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: mul256:
; CHECK-VBMI:       # %bb.0:
; CHECK-VBMI-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-VBMI-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-VBMI-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-VBMI-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-VBMI-NEXT:    vpmullw %ymm3, %ymm1, %ymm4
; CHECK-VBMI-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; CHECK-VBMI-NEXT:    vpandn %ymm3, %ymm5, %ymm3
; CHECK-VBMI-NEXT:    vpmaddubsw %ymm3, %ymm1, %ymm1
; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,32,2,34,4,36,6,38,8,40,10,42,12,44,14,46,16,48,18,50,20,52,22,54,24,56,26,58,28,60,30,62]
; CHECK-VBMI-NEXT:    vpermt2b %ymm1, %ymm3, %ymm4
; CHECK-VBMI-NEXT:    vpmullw %ymm2, %ymm0, %ymm1
; CHECK-VBMI-NEXT:    vpandn %ymm2, %ymm5, %ymm2
; CHECK-VBMI-NEXT:    vpmaddubsw %ymm2, %ymm0, %ymm0
; CHECK-VBMI-NEXT:    vpermt2b %ymm0, %ymm3, %ymm1
; CHECK-VBMI-NEXT:    vmovdqa %ymm1, (%rdx)
; CHECK-VBMI-NEXT:    vmovdqa %ymm4, 32(%rdx)
; CHECK-VBMI-NEXT:    vzeroupper
; CHECK-VBMI-NEXT:    retq
  %d = load <64 x i8>, ptr %a
  %e = load <64 x i8>, ptr %b
  %f = mul <64 x i8> %d, %e
  store <64 x i8> %f, ptr %c
  ret void
}

define dso_local void @mul512(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="512" {
; CHECK-SKX-NOVBMI-LABEL: mul512:
; CHECK-SKX-NOVBMI:       # %bb.0:
; CHECK-SKX-NOVBMI-NEXT:    vmovdqa64 (%rdi), %zmm0
; CHECK-SKX-NOVBMI-NEXT:    vmovdqa64 (%rsi), %zmm1
; CHECK-SKX-NOVBMI-NEXT:    vpmullw %zmm1, %zmm0, %zmm2
; CHECK-SKX-NOVBMI-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; CHECK-SKX-NOVBMI-NEXT:    vpandnq %zmm1, %zmm3, %zmm1
; CHECK-SKX-NOVBMI-NEXT:    vpmaddubsw %zmm1, %zmm0, %zmm0
; CHECK-SKX-NOVBMI-NEXT:    vpsllw $8, %zmm0, %zmm0
; CHECK-SKX-NOVBMI-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & zmm3)
; CHECK-SKX-NOVBMI-NEXT:    vmovdqa64 %zmm0, (%rdx)
; CHECK-SKX-NOVBMI-NEXT:    vzeroupper
; CHECK-SKX-NOVBMI-NEXT:    retq
;
; CHECK-SKX-VBMI-LABEL: mul512:
; CHECK-SKX-VBMI:       # %bb.0:
; CHECK-SKX-VBMI-NEXT:    vmovdqa64 (%rdi), %zmm0
; CHECK-SKX-VBMI-NEXT:    vmovdqa64 (%rsi), %zmm1
; CHECK-SKX-VBMI-NEXT:    vpmullw %zmm1, %zmm0, %zmm2
; CHECK-SKX-VBMI-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; CHECK-SKX-VBMI-NEXT:    vpmaddubsw %zmm1, %zmm0, %zmm0
; CHECK-SKX-VBMI-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,64,2,66,4,68,6,70,8,72,10,74,12,76,14,78,16,80,18,82,20,84,22,86,24,88,26,90,28,92,30,94,32,96,34,98,36,100,38,102,40,104,42,106,44,108,46,110,48,112,50,114,52,116,54,118,56,120,58,122,60,124,62,126]
; CHECK-SKX-VBMI-NEXT:    vpermi2b %zmm0, %zmm2, %zmm1
; CHECK-SKX-VBMI-NEXT:    vmovdqa64 %zmm1, (%rdx)
; CHECK-SKX-VBMI-NEXT:    vzeroupper
; CHECK-SKX-VBMI-NEXT:    retq
;
; CHECK-AVX512-LABEL: mul512:
; CHECK-AVX512:       # %bb.0:
; CHECK-AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
; CHECK-AVX512-NEXT:    vmovdqa64 (%rsi), %zmm1
; CHECK-AVX512-NEXT:    vpmullw %zmm1, %zmm0, %zmm2
; CHECK-AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; CHECK-AVX512-NEXT:    vpandnq %zmm1, %zmm3, %zmm1
; CHECK-AVX512-NEXT:    vpmaddubsw %zmm1, %zmm0, %zmm0
; CHECK-AVX512-NEXT:    vpsllw $8, %zmm0, %zmm0
; CHECK-AVX512-NEXT:    vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & zmm3)
; CHECK-AVX512-NEXT:    vmovdqa64 %zmm0, (%rdx)
; CHECK-AVX512-NEXT:    vzeroupper
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: mul512:
; CHECK-VBMI:       # %bb.0:
; CHECK-VBMI-NEXT:    vmovdqa64 (%rdi), %zmm0
; CHECK-VBMI-NEXT:    vmovdqa64 (%rsi), %zmm1
; CHECK-VBMI-NEXT:    vpmullw %zmm1, %zmm0, %zmm2
; CHECK-VBMI-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; CHECK-VBMI-NEXT:    vpmaddubsw %zmm1, %zmm0, %zmm0
; CHECK-VBMI-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,64,2,66,4,68,6,70,8,72,10,74,12,76,14,78,16,80,18,82,20,84,22,86,24,88,26,90,28,92,30,94,32,96,34,98,36,100,38,102,40,104,42,106,44,108,46,110,48,112,50,114,52,116,54,118,56,120,58,122,60,124,62,126]
; CHECK-VBMI-NEXT:    vpermi2b %zmm0, %zmm2, %zmm1
; CHECK-VBMI-NEXT:    vmovdqa64 %zmm1, (%rdx)
; CHECK-VBMI-NEXT:    vzeroupper
; CHECK-VBMI-NEXT:    retq
  %d = load <64 x i8>, ptr %a
  %e = load <64 x i8>, ptr %b
  %f = mul <64 x i8> %d, %e
  store <64 x i8> %f, ptr %c
  ret void
}

; This threw an assertion at one point.
define <4 x i32> @mload_v4i32(<4 x i32> %trigger, ptr %addr, <4 x i32> %dst) "min-legal-vector-width"="256" {
; CHECK-LABEL: mload_v4i32:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vptestnmd %xmm0, %xmm0, %k1
; CHECK-NEXT:    vpblendmd (%rdi), %xmm1, %xmm0 {%k1}
; CHECK-NEXT:    retq
  %mask = icmp eq <4 x i32> %trigger, zeroinitializer
  %res = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
  ret <4 x i32> %res
}
declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32, <4 x i1>, <4 x i32>)

define <16 x i32> @trunc_v16i64_v16i32(ptr %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i64_v16i32:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vmovdqa 64(%rdi), %ymm2
; CHECK-NEXT:    vmovdqa 96(%rdi), %ymm3
; CHECK-NEXT:    vpmovqd %ymm0, %xmm0
; CHECK-NEXT:    vpmovqd %ymm1, %xmm1
; CHECK-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
; CHECK-NEXT:    vpmovqd %ymm2, %xmm1
; CHECK-NEXT:    vpmovqd %ymm3, %xmm2
; CHECK-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
; CHECK-NEXT:    retq
  %a = load <16 x i64>, ptr %x
  %b = trunc <16 x i64> %a to <16 x i32>
  ret <16 x i32> %b
}

define <16 x i8> @trunc_v16i64_v16i8(ptr %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i64_v16i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vmovdqa 64(%rdi), %ymm2
; CHECK-NEXT:    vmovdqa 96(%rdi), %ymm3
; CHECK-NEXT:    vpmovqb %ymm3, %xmm3
; CHECK-NEXT:    vpmovqb %ymm2, %xmm2
; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; CHECK-NEXT:    vpmovqb %ymm1, %xmm1
; CHECK-NEXT:    vpmovqb %ymm0, %xmm0
; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = load <16 x i64>, ptr %x
  %b = trunc <16 x i64> %a to <16 x i8>
  ret <16 x i8> %b
}

define <16 x i8> @trunc_v16i32_v16i8(ptr %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i32_v16i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vpmovdb %ymm1, %xmm1
; CHECK-NEXT:    vpmovdb %ymm0, %xmm0
; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = load <16 x i32>, ptr %x
  %b = trunc <16 x i32> %a to <16 x i8>
  ret <16 x i8> %b
}

define <8 x i8> @trunc_v8i64_v8i8(ptr %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v8i64_v8i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vpmovqb %ymm1, %xmm1
; CHECK-NEXT:    vpmovqb %ymm0, %xmm0
; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = load <8 x i64>, ptr %x
  %b = trunc <8 x i64> %a to <8 x i8>
  ret <8 x i8> %b
}

define <8 x i16> @trunc_v8i64_v8i16(ptr %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v8i64_v8i16:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vpmovqw %ymm1, %xmm1
; CHECK-NEXT:    vpmovqw %ymm0, %xmm0
; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = load <8 x i64>, ptr %x
  %b = trunc <8 x i64> %a to <8 x i16>
  ret <8 x i16> %b
}

define <8 x i32> @trunc_v8i64_v8i32_zeroes(ptr %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v8i64_v8i32_zeroes:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpsrlq $48, 32(%rdi), %ymm0
; CHECK-NEXT:    vpsrlq $48, (%rdi), %ymm1
; CHECK-NEXT:    vpackusdw %ymm0, %ymm1, %ymm0
; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-NEXT:    retq
  %a = load <8 x i64>, ptr %x
  %b = lshr <8 x i64> %a, <i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48>
  %c = trunc <8 x i64> %b to <8 x i32>
  ret <8 x i32> %c
}

define <16 x i16> @trunc_v16i32_v16i16_zeroes(ptr %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i32_v16i16_zeroes:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm1
; CHECK-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
; CHECK-NEXT:    vpermi2w 32(%rdi), %ymm1, %ymm0
; CHECK-NEXT:    retq
  %a = load <16 x i32>, ptr %x
  %b = lshr <16 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %c = trunc <16 x i32> %b to <16 x i16>
  ret <16 x i16> %c
}

define <32 x i8> @trunc_v32i16_v32i8_zeroes(ptr %x) nounwind "min-legal-vector-width"="256" {
; CHECK-SKX-NOVBMI-LABEL: trunc_v32i16_v32i8_zeroes:
; CHECK-SKX-NOVBMI:       # %bb.0:
; CHECK-SKX-NOVBMI-NEXT:    vpsrlw $8, 32(%rdi), %ymm0
; CHECK-SKX-NOVBMI-NEXT:    vpsrlw $8, (%rdi), %ymm1
; CHECK-SKX-NOVBMI-NEXT:    vpackuswb %ymm0, %ymm1, %ymm0
; CHECK-SKX-NOVBMI-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-SKX-NOVBMI-NEXT:    retq
;
; CHECK-SKX-VBMI-LABEL: trunc_v32i16_v32i8_zeroes:
; CHECK-SKX-VBMI:       # %bb.0:
; CHECK-SKX-VBMI-NEXT:    vmovdqa (%rdi), %ymm1
; CHECK-SKX-VBMI-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
; CHECK-SKX-VBMI-NEXT:    vpermi2b 32(%rdi), %ymm1, %ymm0
; CHECK-SKX-VBMI-NEXT:    retq
;
; CHECK-AVX512-LABEL: trunc_v32i16_v32i8_zeroes:
; CHECK-AVX512:       # %bb.0:
; CHECK-AVX512-NEXT:    vpsrlw $8, 32(%rdi), %ymm0
; CHECK-AVX512-NEXT:    vpsrlw $8, (%rdi), %ymm1
; CHECK-AVX512-NEXT:    vpackuswb %ymm0, %ymm1, %ymm0
; CHECK-AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: trunc_v32i16_v32i8_zeroes:
; CHECK-VBMI:       # %bb.0:
; CHECK-VBMI-NEXT:    vmovdqa (%rdi), %ymm1
; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
; CHECK-VBMI-NEXT:    vpermi2b 32(%rdi), %ymm1, %ymm0
; CHECK-VBMI-NEXT:    retq
  %a = load <32 x i16>, ptr %x
  %b = lshr <32 x i16> %a, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %c = trunc <32 x i16> %b to <32 x i8>
  ret <32 x i8> %c
}

define <8 x i32> @trunc_v8i64_v8i32_sign(ptr %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v8i64_v8i32_sign:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpsraq $48, 32(%rdi), %ymm0
; CHECK-NEXT:    vpsraq $48, (%rdi), %ymm1
; CHECK-NEXT:    vpackssdw %ymm0, %ymm1, %ymm0
; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-NEXT:    retq
  %a = load <8 x i64>, ptr %x
  %b = ashr <8 x i64> %a, <i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48>
  %c = trunc <8 x i64> %b to <8 x i32>
  ret <8 x i32> %c
}

define <16 x i16> @trunc_v16i32_v16i16_sign(ptr %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i32_v16i16_sign:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm1
; CHECK-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
; CHECK-NEXT:    vpermi2w 32(%rdi), %ymm1, %ymm0
; CHECK-NEXT:    retq
  %a = load <16 x i32>, ptr %x
  %b = ashr <16 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %c = trunc <16 x i32> %b to <16 x i16>
  ret <16 x i16> %c
}

define <32 x i8> @trunc_v32i16_v32i8_sign(ptr %x) nounwind "min-legal-vector-width"="256" {
; CHECK-SKX-NOVBMI-LABEL: trunc_v32i16_v32i8_sign:
; CHECK-SKX-NOVBMI:       # %bb.0:
; CHECK-SKX-NOVBMI-NEXT:    vpsrlw $8, 32(%rdi), %ymm0
; CHECK-SKX-NOVBMI-NEXT:    vpsrlw $8, (%rdi), %ymm1
; CHECK-SKX-NOVBMI-NEXT:    vpackuswb %ymm0, %ymm1, %ymm0
; CHECK-SKX-NOVBMI-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-SKX-NOVBMI-NEXT:    retq
;
; CHECK-SKX-VBMI-LABEL: trunc_v32i16_v32i8_sign:
; CHECK-SKX-VBMI:       # %bb.0:
; CHECK-SKX-VBMI-NEXT:    vmovdqa (%rdi), %ymm1
; CHECK-SKX-VBMI-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
; CHECK-SKX-VBMI-NEXT:    vpermi2b 32(%rdi), %ymm1, %ymm0
; CHECK-SKX-VBMI-NEXT:    retq
;
; CHECK-AVX512-LABEL: trunc_v32i16_v32i8_sign:
; CHECK-AVX512:       # %bb.0:
; CHECK-AVX512-NEXT:    vpsrlw $8, 32(%rdi), %ymm0
; CHECK-AVX512-NEXT:    vpsrlw $8, (%rdi), %ymm1
; CHECK-AVX512-NEXT:    vpackuswb %ymm0, %ymm1, %ymm0
; CHECK-AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: trunc_v32i16_v32i8_sign:
; CHECK-VBMI:       # %bb.0:
; CHECK-VBMI-NEXT:    vmovdqa (%rdi), %ymm1
; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
; CHECK-VBMI-NEXT:    vpermi2b 32(%rdi), %ymm1, %ymm0
; CHECK-VBMI-NEXT:    retq
  %a = load <32 x i16>, ptr %x
  %b = ashr <32 x i16> %a, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %c = trunc <32 x i16> %b to <32 x i8>
  ret <32 x i8> %c
}

define dso_local void @zext_v16i8_v16i64(<16 x i8> %x, ptr %y) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: zext_v16i8_v16i64:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; CHECK-NEXT:    vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
; CHECK-NEXT:    vextracti128 $1, %ymm1, %xmm1
; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
; CHECK-NEXT:    vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
; CHECK-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
; CHECK-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
; CHECK-NEXT:    vmovdqa %ymm0, (%rdi)
; CHECK-NEXT:    vmovdqa %ymm1, 64(%rdi)
; CHECK-NEXT:    vmovdqa %ymm3, 96(%rdi)
; CHECK-NEXT:    vmovdqa %ymm2, 32(%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = zext <16 x i8> %x to <16 x i64>
  store <16 x i64> %a, ptr %y
  ret void
}

define dso_local void @sext_v16i8_v16i64(<16 x i8> %x, ptr %y) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: sext_v16i8_v16i64:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovsxbw %xmm0, %ymm1
; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; CHECK-NEXT:    vpmovsxwq %xmm2, %ymm2
; CHECK-NEXT:    vextracti128 $1, %ymm1, %xmm1
; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
; CHECK-NEXT:    vpmovsxwq %xmm3, %ymm3
; CHECK-NEXT:    vpmovsxwq %xmm1, %ymm1
; CHECK-NEXT:    vpmovsxbq %xmm0, %ymm0
; CHECK-NEXT:    vmovdqa %ymm0, (%rdi)
; CHECK-NEXT:    vmovdqa %ymm1, 64(%rdi)
; CHECK-NEXT:    vmovdqa %ymm3, 96(%rdi)
; CHECK-NEXT:    vmovdqa %ymm2, 32(%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = sext <16 x i8> %x to <16 x i64>
  store <16 x i64> %a, ptr %y
  ret void
}

define dso_local void @vselect_split_v8i16_setcc(<8 x i16> %s, <8 x i16> %t, ptr %p, ptr %q, ptr %r) "min-legal-vector-width"="256" {
; CHECK-LABEL: vselect_split_v8i16_setcc:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-NEXT:    vpcmpeqw %xmm1, %xmm0, %k1
; CHECK-NEXT:    kshiftrb $4, %k1, %k2
; CHECK-NEXT:    vmovdqa64 32(%rdi), %ymm3 {%k2}
; CHECK-NEXT:    vmovdqa64 (%rdi), %ymm2 {%k1}
; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <8 x i64>, ptr %p
  %y = load <8 x i64>, ptr %q
  %a = icmp eq <8 x i16> %s, %t
  %b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y
  store <8 x i64> %b, ptr %r
  ret void
}

define dso_local void @vselect_split_v8i32_setcc(<8 x i32> %s, <8 x i32> %t, ptr %p, ptr %q, ptr %r) "min-legal-vector-width"="256" {
; CHECK-LABEL: vselect_split_v8i32_setcc:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-NEXT:    vpcmpeqd %ymm1, %ymm0, %k1
; CHECK-NEXT:    kshiftrb $4, %k1, %k2
; CHECK-NEXT:    vmovdqa64 32(%rdi), %ymm3 {%k2}
; CHECK-NEXT:    vmovdqa64 (%rdi), %ymm2 {%k1}
; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <8 x i64>, ptr %p
  %y = load <8 x i64>, ptr %q
  %a = icmp eq <8 x i32> %s, %t
  %b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y
  store <8 x i64> %b, ptr %r
  ret void
}

define dso_local void @vselect_split_v16i8_setcc(<16 x i8> %s, <16 x i8> %t, ptr %p, ptr %q, ptr %r) "min-legal-vector-width"="256" {
; CHECK-LABEL: vselect_split_v16i8_setcc:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-NEXT:    vpcmpeqb %xmm1, %xmm0, %k1
; CHECK-NEXT:    kshiftrw $8, %k1, %k2
; CHECK-NEXT:    vmovdqa32 32(%rdi), %ymm3 {%k2}
; CHECK-NEXT:    vmovdqa32 (%rdi), %ymm2 {%k1}
; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <16 x i32>, ptr %p
  %y = load <16 x i32>, ptr %q
  %a = icmp eq <16 x i8> %s, %t
  %b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y
  store <16 x i32> %b, ptr %r
  ret void
}

define dso_local void @vselect_split_v16i16_setcc(<16 x i16> %s, <16 x i16> %t, ptr %p, ptr %q, ptr %r) "min-legal-vector-width"="256" {
; CHECK-LABEL: vselect_split_v16i16_setcc:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-NEXT:    vpcmpeqw %ymm1, %ymm0, %k1
; CHECK-NEXT:    kshiftrw $8, %k1, %k2
; CHECK-NEXT:    vmovdqa32 32(%rdi), %ymm3 {%k2}
; CHECK-NEXT:    vmovdqa32 (%rdi), %ymm2 {%k1}
; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <16 x i32>, ptr %p
  %y = load <16 x i32>, ptr %q
  %a = icmp eq <16 x i16> %s, %t
  %b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y
  store <16 x i32> %b, ptr %r
  ret void
}

define <16 x i8> @trunc_packus_v16i32_v16i8(ptr %p) "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_packus_v16i32_v16i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vpackusdw 32(%rdi), %ymm0, %ymm0
; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-NEXT:    vpmovuswb %ymm0, %xmm0
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = load <16 x i32>, ptr %p
  %b = icmp slt <16 x i32> %a, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
  %c = select <16 x i1> %b, <16 x i32> %a, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
  %d = icmp sgt <16 x i32> %c, zeroinitializer
  %e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> zeroinitializer
  %f = trunc <16 x i32> %e to <16 x i8>
  ret <16 x i8> %f
}

define dso_local void @trunc_packus_v16i32_v16i8_store(ptr %p, ptr %q) "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_packus_v16i32_v16i8_store:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vpackusdw 32(%rdi), %ymm0, %ymm0
; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-NEXT:    vpmovuswb %ymm0, (%rsi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = load <16 x i32>, ptr %p
  %b = icmp slt <16 x i32> %a, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
  %c = select <16 x i1> %b, <16 x i32> %a, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
  %d = icmp sgt <16 x i32> %c, zeroinitializer
  %e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> zeroinitializer
  %f = trunc <16 x i32> %e to <16 x i8>
  store <16 x i8> %f, ptr %q
  ret void
}

define <64 x i1> @v64i1_argument_return(<64 x i1> %x) "min-legal-vector-width"="256" {
; CHECK-LABEL: v64i1_argument_return:
; CHECK:       # %bb.0:
; CHECK-NEXT:    retq
  ret <64 x i1> %x
}

define dso_local void @v64i1_shuffle(ptr %x, ptr %y) "min-legal-vector-width"="256" {
; CHECK-LABEL: v64i1_shuffle:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vmovdqa (%rdi), %ymm1
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm0
; CHECK-NEXT:    vptestnmb %ymm1, %ymm1, %k0
; CHECK-NEXT:    kshiftrd $1, %k0, %k1
; CHECK-NEXT:    kshiftlq $63, %k0, %k2
; CHECK-NEXT:    kshiftrq $62, %k2, %k2
; CHECK-NEXT:    kshiftlq $63, %k1, %k1
; CHECK-NEXT:    kshiftrq $63, %k1, %k1
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-5, %rax
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $3, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $61, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-9, %rax
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $2, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $60, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-17, %rax
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $5, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $59, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-33, %rax
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $4, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $58, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-65, %rax
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $7, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $57, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-129, %rax
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $6, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $56, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-257, %rax # imm = 0xFEFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $9, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $55, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-513, %rax # imm = 0xFDFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $8, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $54, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-1025, %rax # imm = 0xFBFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $11, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $53, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-2049, %rax # imm = 0xF7FF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $10, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $52, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-4097, %rax # imm = 0xEFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $13, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $51, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-8193, %rax # imm = 0xDFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $12, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $50, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-16385, %rax # imm = 0xBFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $15, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $49, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-32769, %rax # imm = 0xFFFF7FFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $14, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $48, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-65537, %rax # imm = 0xFFFEFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $17, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $47, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-131073, %rax # imm = 0xFFFDFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $16, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $46, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-262145, %rax # imm = 0xFFFBFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $19, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $45, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-524289, %rax # imm = 0xFFF7FFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $18, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $44, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-1048577, %rax # imm = 0xFFEFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $21, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $43, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-2097153, %rax # imm = 0xFFDFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $20, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $42, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-4194305, %rax # imm = 0xFFBFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $23, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $41, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-8388609, %rax # imm = 0xFF7FFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $22, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $40, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-16777217, %rax # imm = 0xFEFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $25, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $39, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-33554433, %rax # imm = 0xFDFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $24, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $38, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-67108865, %rax # imm = 0xFBFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $27, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $37, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-134217729, %rax # imm = 0xF7FFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $26, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $36, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-268435457, %rax # imm = 0xEFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $29, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $35, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-536870913, %rax # imm = 0xDFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $28, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $34, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-1073741825, %rax # imm = 0xBFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $31, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $33, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movabsq $-2147483649, %rax # imm = 0xFFFFFFFF7FFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k2
; CHECK-NEXT:    vptestnmb %ymm0, %ymm0, %k1
; CHECK-NEXT:    kshiftrd $30, %k0, %k0
; CHECK-NEXT:    kshiftlq $63, %k0, %k0
; CHECK-NEXT:    kshiftrq $32, %k0, %k0
; CHECK-NEXT:    korq %k0, %k2, %k0
; CHECK-NEXT:    movabsq $-4294967297, %rax # imm = 0xFFFFFFFEFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $1, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $31, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-8589934593, %rax # imm = 0xFFFFFFFDFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftlq $63, %k1, %k2
; CHECK-NEXT:    kshiftrq $30, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-17179869185, %rax # imm = 0xFFFFFFFBFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $3, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $29, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-34359738369, %rax # imm = 0xFFFFFFF7FFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $2, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $28, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-68719476737, %rax # imm = 0xFFFFFFEFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $5, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $27, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-137438953473, %rax # imm = 0xFFFFFFDFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $4, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $26, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-274877906945, %rax # imm = 0xFFFFFFBFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $7, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $25, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-549755813889, %rax # imm = 0xFFFFFF7FFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $6, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $24, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-1099511627777, %rax # imm = 0xFFFFFEFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $9, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $23, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-2199023255553, %rax # imm = 0xFFFFFDFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $8, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $22, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-4398046511105, %rax # imm = 0xFFFFFBFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $11, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $21, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-8796093022209, %rax # imm = 0xFFFFF7FFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $10, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $20, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-17592186044417, %rax # imm = 0xFFFFEFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $13, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $19, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-35184372088833, %rax # imm = 0xFFFFDFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $12, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $18, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-70368744177665, %rax # imm = 0xFFFFBFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $15, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $17, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-140737488355329, %rax # imm = 0xFFFF7FFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $14, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $16, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-281474976710657, %rax # imm = 0xFFFEFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $17, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $15, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-562949953421313, %rax # imm = 0xFFFDFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $16, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $14, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-1125899906842625, %rax # imm = 0xFFFBFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $19, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $13, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-2251799813685249, %rax # imm = 0xFFF7FFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $18, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $12, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-4503599627370497, %rax # imm = 0xFFEFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $21, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $11, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-9007199254740993, %rax # imm = 0xFFDFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $20, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $10, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-18014398509481985, %rax # imm = 0xFFBFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $23, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $9, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-36028797018963969, %rax # imm = 0xFF7FFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $22, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $8, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-72057594037927937, %rax # imm = 0xFEFFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $25, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $7, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-144115188075855873, %rax # imm = 0xFDFFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $24, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $6, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-288230376151711745, %rax # imm = 0xFBFFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $27, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $5, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-576460752303423489, %rax # imm = 0xF7FFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $26, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $4, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-1152921504606846977, %rax # imm = 0xEFFFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $29, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $3, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-2305843009213693953, %rax # imm = 0xDFFFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $28, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $2, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-4611686018427387905, %rax # imm = 0xBFFFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $31, %k1, %k2
; CHECK-NEXT:    kshiftlq $62, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $30, %k1, %k1
; CHECK-NEXT:    kshiftlq $1, %k0, %k0
; CHECK-NEXT:    kshiftrq $1, %k0, %k0
; CHECK-NEXT:    kshiftlq $63, %k1, %k1
; CHECK-NEXT:    korq %k1, %k0, %k1
; CHECK-NEXT:    vmovdqu8 %ymm1, (%rsi) {%k1}
; CHECK-NEXT:    kshiftrq $32, %k1, %k1
; CHECK-NEXT:    vmovdqu8 %ymm0, 32(%rsi) {%k1}
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
entry:
  %a = load <64 x i8>, ptr %x
  %b = icmp eq <64 x i8> %a, zeroinitializer
  %shuf = shufflevector <64 x i1> %b, <64 x i1> undef, <64 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30, i32 33, i32 32, i32 35, i32 34, i32 37, i32 36, i32 39, i32 38, i32 41, i32 40, i32 43, i32 42, i32 45, i32 44, i32 47, i32 46, i32 49, i32 48, i32 51, i32 50, i32 53, i32 52, i32 55, i32 54, i32 57, i32 56, i32 59, i32 58, i32 61, i32 60, i32 63, i32 62>
  call void @llvm.masked.store.v64i8.p0(<64 x i8> %a, ptr %y, i32 1, <64 x i1> %shuf)
  ret void
}
declare void @llvm.masked.store.v64i8.p0(<64 x i8>, ptr, i32, <64 x i1>)

@mem64_dst = dso_local global i64 0, align 8
@mem64_src = dso_local global i64 0, align 8
define dso_local i32 @v64i1_inline_asm() "min-legal-vector-width"="256" {
; CHECK-LABEL: v64i1_inline_asm:
; CHECK:       # %bb.0:
; CHECK-NEXT:    kmovq mem64_src(%rip), %k0
; CHECK-NEXT:    #APP
; CHECK-NEXT:    #NO_APP
; CHECK-NEXT:    kmovq %k0, mem64_dst(%rip)
; CHECK-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
; CHECK-NEXT:    retq
  %1 = alloca i32, align 4
  %2 = load i64, ptr @mem64_src, align 8
  %3 = call i64 asm "", "=k,k,~{dirflag},~{fpsr},~{flags}"(i64 %2)
  store i64 %3, ptr @mem64_dst, align 8
  %4 = load i32, ptr %1, align 4
  ret i32 %4
}

define dso_local void @cmp_v8i64_sext(ptr %xptr, ptr %yptr, ptr %zptr) "min-legal-vector-width"="256" {
; CHECK-LABEL: cmp_v8i64_sext:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rsi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm1
; CHECK-NEXT:    vpcmpgtq 32(%rdi), %ymm1, %ymm1
; CHECK-NEXT:    vpcmpgtq (%rdi), %ymm0, %ymm0
; CHECK-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <8 x i64>, ptr %xptr
  %y = load <8 x i64>, ptr %yptr
  %cmp = icmp slt <8 x i64> %x, %y
  %ext = sext <8 x i1> %cmp to <8 x i64>
  store <8 x i64> %ext, ptr %zptr
  ret void
}

define dso_local void @cmp_v8i64_zext(ptr %xptr, ptr %yptr, ptr %zptr) "min-legal-vector-width"="256" {
; CHECK-LABEL: cmp_v8i64_zext:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rsi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm1
; CHECK-NEXT:    vpcmpgtq 32(%rdi), %ymm1, %ymm1
; CHECK-NEXT:    vpcmpgtq (%rdi), %ymm0, %ymm0
; CHECK-NEXT:    vpsrlq $63, %ymm1, %ymm1
; CHECK-NEXT:    vpsrlq $63, %ymm0, %ymm0
; CHECK-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <8 x i64>, ptr %xptr
  %y = load <8 x i64>, ptr %yptr
  %cmp = icmp slt <8 x i64> %x, %y
  %ext = zext <8 x i1> %cmp to <8 x i64>
  store <8 x i64> %ext, ptr %zptr
  ret void
}

define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: var_rotate_v16i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm1
; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2
; CHECK-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
; CHECK-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; CHECK-NEXT:    vpsllvw %xmm2, %xmm3, %xmm2
; CHECK-NEXT:    vpsrlw $8, %xmm2, %xmm2
; CHECK-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
; CHECK-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; CHECK-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0
; CHECK-NEXT:    vpsrlw $8, %xmm0, %xmm0
; CHECK-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
; CHECK-NEXT:    retq
  %b8 = sub <16 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %b
  %shl = shl <16 x i8> %a, %b
  %lshr = lshr <16 x i8> %a, %b8
  %or = or <16 x i8> %shl, %lshr
  ret <16 x i8> %or
}

define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: var_rotate_v32i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2
; CHECK-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; CHECK-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
; CHECK-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-NEXT:    vpsllvw %ymm3, %ymm4, %ymm3
; CHECK-NEXT:    vpsrlw $8, %ymm3, %ymm3
; CHECK-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
; CHECK-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
; CHECK-NEXT:    vpsrlw $8, %ymm0, %ymm0
; CHECK-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
; CHECK-NEXT:    retq
  %b8 = sub <32 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %b
  %shl = shl <32 x i8> %a, %b
  %lshr = lshr <32 x i8> %a, %b8
  %or = or <32 x i8> %shl, %lshr
  ret <32 x i8> %or
}

define <32 x i8> @splatvar_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: splatvar_rotate_v32i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
; CHECK-NEXT:    vpsllw %xmm1, %ymm2, %ymm2
; CHECK-NEXT:    vpsrlw $8, %ymm2, %ymm2
; CHECK-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
; CHECK-NEXT:    vpsrlw $8, %ymm0, %ymm0
; CHECK-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
; CHECK-NEXT:    retq
  %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
  %splat8 = sub <32 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %splat
  %shl = shl <32 x i8> %a, %splat
  %lshr = lshr <32 x i8> %a, %splat8
  %or = or <32 x i8> %shl, %lshr
  ret <32 x i8> %or
}

define <32 x i8> @constant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: constant_rotate_v32i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-NEXT:    vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; CHECK-NEXT:    vpsrlw $8, %ymm1, %ymm1
; CHECK-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-NEXT:    vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; CHECK-NEXT:    vpsrlw $8, %ymm0, %ymm0
; CHECK-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
; CHECK-NEXT:    retq
  %shl = shl <32 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>
  %lshr = lshr <32 x i8> %a, <i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
  %or = or <32 x i8> %shl, %lshr
  ret <32 x i8> %or
}

define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
; CHECK-SKX-LABEL: splatconstant_rotate_v32i8:
; CHECK-SKX:       # %bb.0:
; CHECK-SKX-NEXT:    vpsllw $4, %ymm0, %ymm1
; CHECK-SKX-NEXT:    vpsrlw $4, %ymm0, %ymm0
; CHECK-SKX-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
; CHECK-SKX-NEXT:    retq
;
; CHECK-AVX512-LABEL: splatconstant_rotate_v32i8:
; CHECK-AVX512:       # %bb.0:
; CHECK-AVX512-NEXT:    vpsllw $4, %ymm0, %ymm1
; CHECK-AVX512-NEXT:    vpsrlw $4, %ymm0, %ymm0
; CHECK-AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI1-LABEL: splatconstant_rotate_v32i8:
; CHECK-VBMI1:       # %bb.0:
; CHECK-VBMI1-NEXT:    vpsllw $4, %ymm0, %ymm1
; CHECK-VBMI1-NEXT:    vpsrlw $4, %ymm0, %ymm0
; CHECK-VBMI1-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
; CHECK-VBMI1-NEXT:    retq
;
; CHECK-GFNI-LABEL: splatconstant_rotate_v32i8:
; CHECK-GFNI:       # %bb.0:
; CHECK-GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0 # [8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16]
; CHECK-GFNI-NEXT:    retq
  %shl = shl <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
  %lshr = lshr <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
  %or = or <32 x i8> %shl, %lshr
  ret <32 x i8> %or
}

define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
; CHECK-SKX-LABEL: splatconstant_rotate_mask_v32i8:
; CHECK-SKX:       # %bb.0:
; CHECK-SKX-NEXT:    vpsllw $4, %ymm0, %ymm1
; CHECK-SKX-NEXT:    vpsrlw $4, %ymm0, %ymm0
; CHECK-SKX-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
; CHECK-SKX-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
; CHECK-SKX-NEXT:    retq
;
; CHECK-AVX512-LABEL: splatconstant_rotate_mask_v32i8:
; CHECK-AVX512:       # %bb.0:
; CHECK-AVX512-NEXT:    vpsllw $4, %ymm0, %ymm1
; CHECK-AVX512-NEXT:    vpsrlw $4, %ymm0, %ymm0
; CHECK-AVX512-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
; CHECK-AVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI1-LABEL: splatconstant_rotate_mask_v32i8:
; CHECK-VBMI1:       # %bb.0:
; CHECK-VBMI1-NEXT:    vpsllw $4, %ymm0, %ymm1
; CHECK-VBMI1-NEXT:    vpsrlw $4, %ymm0, %ymm0
; CHECK-VBMI1-NEXT:    vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))
; CHECK-VBMI1-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
; CHECK-VBMI1-NEXT:    retq
;
; CHECK-GFNI-LABEL: splatconstant_rotate_mask_v32i8:
; CHECK-GFNI:       # %bb.0:
; CHECK-GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0 # [8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16,8,4,2,1,128,64,32,16]
; CHECK-GFNI-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
; CHECK-GFNI-NEXT:    retq
  %shl = shl <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
  %lshr = lshr <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
  %rmask = and <32 x i8> %lshr, <i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55>
  %lmask = and <32 x i8> %shl, <i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33>
  %or = or <32 x i8> %lmask, %rmask
  ret <32 x i8> %or
}

define <8 x i64> @udiv_v8i64_prefer256(<8 x i64> %x, <8 x i64> %y) nounwind "min-legal-vector-width"="0" {
; CHECK-LABEL: udiv_v8i64_prefer256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm4
; CHECK-NEXT:    vpextrq $1, %xmm4, %rax
; CHECK-NEXT:    vextracti128 $1, %ymm2, %xmm5
; CHECK-NEXT:    vpextrq $1, %xmm5, %rcx
; CHECK-NEXT:    xorl %edx, %edx
; CHECK-NEXT:    divq %rcx
; CHECK-NEXT:    movq %rax, %rdi
; CHECK-NEXT:    vmovq %xmm4, %rax
; CHECK-NEXT:    vmovq %xmm5, %rcx
; CHECK-NEXT:    xorl %edx, %edx
; CHECK-NEXT:    divq %rcx
; CHECK-NEXT:    movq %rax, %rcx
; CHECK-NEXT:    vpextrq $1, %xmm0, %rax
; CHECK-NEXT:    vpextrq $1, %xmm2, %rsi
; CHECK-NEXT:    xorl %edx, %edx
; CHECK-NEXT:    divq %rsi
; CHECK-NEXT:    movq %rax, %rsi
; CHECK-NEXT:    vmovq %xmm0, %rax
; CHECK-NEXT:    vmovq %xmm2, %r8
; CHECK-NEXT:    xorl %edx, %edx
; CHECK-NEXT:    divq %r8
; CHECK-NEXT:    movq %rax, %r8
; CHECK-NEXT:    vmovq %rdi, %xmm0
; CHECK-NEXT:    vextracti128 $1, %ymm1, %xmm2
; CHECK-NEXT:    vpextrq $1, %xmm2, %rax
; CHECK-NEXT:    vextracti128 $1, %ymm3, %xmm4
; CHECK-NEXT:    vpextrq $1, %xmm4, %rdi
; CHECK-NEXT:    xorl %edx, %edx
; CHECK-NEXT:    divq %rdi
; CHECK-NEXT:    movq %rax, %rdi
; CHECK-NEXT:    vmovq %rcx, %xmm5
; CHECK-NEXT:    vmovq %xmm2, %rax
; CHECK-NEXT:    vmovq %xmm4, %rcx
; CHECK-NEXT:    xorl %edx, %edx
; CHECK-NEXT:    divq %rcx
; CHECK-NEXT:    movq %rax, %rcx
; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm5[0],xmm0[0]
; CHECK-NEXT:    vmovq %rsi, %xmm2
; CHECK-NEXT:    vmovq %r8, %xmm4
; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm4[0],xmm2[0]
; CHECK-NEXT:    vpextrq $1, %xmm1, %rax
; CHECK-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
; CHECK-NEXT:    vpextrq $1, %xmm3, %rsi
; CHECK-NEXT:    xorl %edx, %edx
; CHECK-NEXT:    divq %rsi
; CHECK-NEXT:    movq %rax, %rsi
; CHECK-NEXT:    vmovq %rdi, %xmm2
; CHECK-NEXT:    vmovq %xmm1, %rax
; CHECK-NEXT:    vmovq %xmm3, %rdi
; CHECK-NEXT:    xorl %edx, %edx
; CHECK-NEXT:    divq %rdi
; CHECK-NEXT:    vmovq %rcx, %xmm1
; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; CHECK-NEXT:    vmovq %rsi, %xmm2
; CHECK-NEXT:    vmovq %rax, %xmm3
; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; CHECK-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
; CHECK-NEXT:    retq
  %r = udiv <8 x i64> %x, %y
  ret <8 x i64> %r
}

define <8 x i64> @udiv_v8i64_512(<8 x i64> %x, <8 x i64> %y) nounwind "min-legal-vector-width"="512" {
; CHECK-LABEL: udiv_v8i64_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vcvtuqq2pd {ru-sae}, %zmm1, %zmm2
; CHECK-NEXT:    vbroadcastsd {{.*#+}} zmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
; CHECK-NEXT:    vdivpd {rd-sae}, %zmm2, %zmm3, %zmm2
; CHECK-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm3
; CHECK-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm3, %zmm3
; CHECK-NEXT:    vcvtpd2uqq {rd-sae}, %zmm3, %zmm3
; CHECK-NEXT:    vpmullq %zmm1, %zmm3, %zmm4
; CHECK-NEXT:    vpsubq %zmm4, %zmm0, %zmm0
; CHECK-NEXT:    vcvtuqq2pd {rd-sae}, %zmm0, %zmm4
; CHECK-NEXT:    vmulpd {rd-sae}, %zmm2, %zmm4, %zmm2
; CHECK-NEXT:    vcvtpd2uqq {rd-sae}, %zmm2, %zmm2
; CHECK-NEXT:    vpmullq %zmm1, %zmm2, %zmm4
; CHECK-NEXT:    vpaddq %zmm2, %zmm3, %zmm2
; CHECK-NEXT:    vpsubq %zmm4, %zmm0, %zmm0
; CHECK-NEXT:    vpcmpnltuq %zmm1, %zmm0, %k0
; CHECK-NEXT:    vpmovm2q %k0, %zmm0
; CHECK-NEXT:    vpsubq %zmm0, %zmm2, %zmm0
; CHECK-NEXT:    retq
  %r = udiv <8 x i64> %x, %y
  ret <8 x i64> %r
}