; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -mattr=prefer-256-bit | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -mattr=prefer-256-bit,avx512vbmi | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
; Make sure CPUs default to prefer-256-bit. avx512vnni isn't interesting as it just adds an isel peephole for vpmaddwd+vpaddd
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cascadelake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cooperlake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=cannonlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-client | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-server | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI

; This file primarily contains tests for specific places in X86ISelLowering.cpp that needed be made aware of the legalizer not allowing 512-bit vectors due to prefer-256-bit even though AVX512 is enabled.

define dso_local void @add256(<16 x i32>* %a, <16 x i32>* %b, <16 x i32>* %c) "min-legal-vector-width"="256" {
; CHECK-LABEL: add256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vpaddd 32(%rsi), %ymm1, %ymm1
; CHECK-NEXT:    vpaddd (%rsi), %ymm0, %ymm0
; CHECK-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %d = load <16 x i32>, <16 x i32>* %a
  %e = load <16 x i32>, <16 x i32>* %b
  %f = add <16 x i32> %d, %e
  store <16 x i32> %f, <16 x i32>* %c
  ret void
}

define dso_local void @add512(<16 x i32>* %a, <16 x i32>* %b, <16 x i32>* %c) "min-legal-vector-width"="512" {
; CHECK-LABEL: add512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
; CHECK-NEXT:    vpaddd (%rsi), %zmm0, %zmm0
; CHECK-NEXT:    vmovdqa64 %zmm0, (%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %d = load <16 x i32>, <16 x i32>* %a
  %e = load <16 x i32>, <16 x i32>* %b
  %f = add <16 x i32> %d, %e
  store <16 x i32> %f, <16 x i32>* %c
  ret void
}

define dso_local void @avg_v64i8_256(<64 x i8>* %a, <64 x i8>* %b) "min-legal-vector-width"="256" {
; CHECK-LABEL: avg_v64i8_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rsi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm1
; CHECK-NEXT:    vpavgb (%rdi), %ymm0, %ymm0
; CHECK-NEXT:    vpavgb 32(%rdi), %ymm1, %ymm1
; CHECK-NEXT:    vmovdqu %ymm1, (%rax)
; CHECK-NEXT:    vmovdqu %ymm0, (%rax)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %1 = load <64 x i8>, <64 x i8>* %a
  %2 = load <64 x i8>, <64 x i8>* %b
  %3 = zext <64 x i8> %1 to <64 x i32>
  %4 = zext <64 x i8> %2 to <64 x i32>
  %5 = add nuw nsw <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %6 = add nuw nsw <64 x i32> %5, %4
  %7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %8 = trunc <64 x i32> %7 to <64 x i8>
  store <64 x i8> %8, <64 x i8>* undef, align 4
  ret void
}


define dso_local void @avg_v64i8_512(<64 x i8>* %a, <64 x i8>* %b) "min-legal-vector-width"="512" {
; CHECK-LABEL: avg_v64i8_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm0
; CHECK-NEXT:    vpavgb (%rdi), %zmm0, %zmm0
; CHECK-NEXT:    vmovdqu64 %zmm0, (%rax)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %1 = load <64 x i8>, <64 x i8>* %a
  %2 = load <64 x i8>, <64 x i8>* %b
  %3 = zext <64 x i8> %1 to <64 x i32>
  %4 = zext <64 x i8> %2 to <64 x i32>
  %5 = add nuw nsw <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %6 = add nuw nsw <64 x i32> %5, %4
  %7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %8 = trunc <64 x i32> %7 to <64 x i8>
  store <64 x i8> %8, <64 x i8>* undef, align 4
  ret void
}

define dso_local void @pmaddwd_32_256(<32 x i16>* %APtr, <32 x i16>* %BPtr, <16 x i32>* %CPtr) "min-legal-vector-width"="256" {
; CHECK-LABEL: pmaddwd_32_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vpmaddwd 32(%rsi), %ymm1, %ymm1
; CHECK-NEXT:    vpmaddwd (%rsi), %ymm0, %ymm0
; CHECK-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
   %A = load <32 x i16>, <32 x i16>* %APtr
   %B = load <32 x i16>, <32 x i16>* %BPtr
   %a = sext <32 x i16> %A to <32 x i32>
   %b = sext <32 x i16> %B to <32 x i32>
   %m = mul nsw <32 x i32> %a, %b
   %odd = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
   %even = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
   %ret = add <16 x i32> %odd, %even
   store <16 x i32> %ret, <16 x i32>* %CPtr
   ret void
}

define dso_local void @pmaddwd_32_512(<32 x i16>* %APtr, <32 x i16>* %BPtr, <16 x i32>* %CPtr) "min-legal-vector-width"="512" {
; CHECK-LABEL: pmaddwd_32_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
; CHECK-NEXT:    vpmaddwd (%rsi), %zmm0, %zmm0
; CHECK-NEXT:    vmovdqa64 %zmm0, (%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
   %A = load <32 x i16>, <32 x i16>* %APtr
   %B = load <32 x i16>, <32 x i16>* %BPtr
   %a = sext <32 x i16> %A to <32 x i32>
   %b = sext <32 x i16> %B to <32 x i32>
   %m = mul nsw <32 x i32> %a, %b
   %odd = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
   %even = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
   %ret = add <16 x i32> %odd, %even
   store <16 x i32> %ret, <16 x i32>* %CPtr
   ret void
}

define dso_local void @psubus_64i8_max_256(<64 x i8>* %xptr, <64 x i8>* %yptr, <64 x i8>* %zptr) "min-legal-vector-width"="256" {
; CHECK-LABEL: psubus_64i8_max_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vpsubusb 32(%rsi), %ymm1, %ymm1
; CHECK-NEXT:    vpsubusb (%rsi), %ymm0, %ymm0
; CHECK-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <64 x i8>, <64 x i8>* %xptr
  %y = load <64 x i8>, <64 x i8>* %yptr
  %cmp = icmp ult <64 x i8> %x, %y
  %max = select <64 x i1> %cmp, <64 x i8> %y, <64 x i8> %x
  %res = sub <64 x i8> %max, %y
  store <64 x i8> %res, <64 x i8>* %zptr
  ret void
}

define dso_local void @psubus_64i8_max_512(<64 x i8>* %xptr, <64 x i8>* %yptr, <64 x i8>* %zptr) "min-legal-vector-width"="512" {
; CHECK-LABEL: psubus_64i8_max_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0
; CHECK-NEXT:    vpsubusb (%rsi), %zmm0, %zmm0
; CHECK-NEXT:    vmovdqa64 %zmm0, (%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <64 x i8>, <64 x i8>* %xptr
  %y = load <64 x i8>, <64 x i8>* %yptr
  %cmp = icmp ult <64 x i8> %x, %y
  %max = select <64 x i1> %cmp, <64 x i8> %y, <64 x i8> %x
  %res = sub <64 x i8> %max, %y
  store <64 x i8> %res, <64 x i8>* %zptr
  ret void
}

define dso_local i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly, i32) "min-legal-vector-width"="256" {
; CHECK-LABEL: _Z9test_charPcS_i_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    movl %edx, %eax
; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT:    xorl %ecx, %ecx
; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB8_1: # %vector.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vpmovsxbw 16(%rdi,%rcx), %ymm3
; CHECK-NEXT:    vpmovsxbw (%rdi,%rcx), %ymm4
; CHECK-NEXT:    vpmovsxbw 16(%rsi,%rcx), %ymm5
; CHECK-NEXT:    vpmaddwd %ymm3, %ymm5, %ymm3
; CHECK-NEXT:    vpaddd %ymm2, %ymm3, %ymm2
; CHECK-NEXT:    vpmovsxbw (%rsi,%rcx), %ymm3
; CHECK-NEXT:    vpmaddwd %ymm4, %ymm3, %ymm3
; CHECK-NEXT:    vpaddd %ymm1, %ymm3, %ymm1
; CHECK-NEXT:    addq $32, %rcx
; CHECK-NEXT:    cmpq %rcx, %rax
; CHECK-NEXT:    jne .LBB8_1
; CHECK-NEXT:  # %bb.2: # %middle.block
; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm1
; CHECK-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT:    vmovd %xmm0, %eax
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
entry:
  %3 = zext i32 %2 to i64
  br label %vector.body

vector.body:
  %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
  %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
  %4 = getelementptr inbounds i8, i8* %0, i64 %index
  %5 = bitcast i8* %4 to <32 x i8>*
  %wide.load = load <32 x i8>, <32 x i8>* %5, align 1
  %6 = sext <32 x i8> %wide.load to <32 x i32>
  %7 = getelementptr inbounds i8, i8* %1, i64 %index
  %8 = bitcast i8* %7 to <32 x i8>*
  %wide.load14 = load <32 x i8>, <32 x i8>* %8, align 1
  %9 = sext <32 x i8> %wide.load14 to <32 x i32>
  %10 = mul nsw <32 x i32> %9, %6
  %11 = add nsw <32 x i32> %10, %vec.phi
  %index.next = add i64 %index, 32
  %12 = icmp eq i64 %index.next, %3
  br i1 %12, label %middle.block, label %vector.body

middle.block:
  %rdx.shuf1 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx1 = add <32 x i32> %11, %rdx.shuf1
  %rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf
  %rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx32 = add <32 x i32> %bin.rdx, %rdx.shuf15
  %rdx.shuf17 = shufflevector <32 x i32> %bin.rdx32, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx18 = add <32 x i32> %bin.rdx32, %rdx.shuf17
  %rdx.shuf19 = shufflevector <32 x i32> %bin.rdx18, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx20 = add <32 x i32> %bin.rdx18, %rdx.shuf19
  %13 = extractelement <32 x i32> %bin.rdx20, i32 0
  ret i32 %13
}

define dso_local i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly, i32) "min-legal-vector-width"="512" {
; CHECK-LABEL: _Z9test_charPcS_i_512:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    movl %edx, %eax
; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT:    xorl %ecx, %ecx
; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB9_1: # %vector.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vpmovsxbw (%rdi,%rcx), %zmm2
; CHECK-NEXT:    vpmovsxbw (%rsi,%rcx), %zmm3
; CHECK-NEXT:    vpmaddwd %zmm2, %zmm3, %zmm2
; CHECK-NEXT:    vpaddd %zmm1, %zmm2, %zmm1
; CHECK-NEXT:    addq $32, %rcx
; CHECK-NEXT:    cmpq %rcx, %rax
; CHECK-NEXT:    jne .LBB9_1
; CHECK-NEXT:  # %bb.2: # %middle.block
; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
; CHECK-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT:    vmovd %xmm0, %eax
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
entry:
  %3 = zext i32 %2 to i64
  br label %vector.body

vector.body:
  %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]
  %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]
  %4 = getelementptr inbounds i8, i8* %0, i64 %index
  %5 = bitcast i8* %4 to <32 x i8>*
  %wide.load = load <32 x i8>, <32 x i8>* %5, align 1
  %6 = sext <32 x i8> %wide.load to <32 x i32>
  %7 = getelementptr inbounds i8, i8* %1, i64 %index
  %8 = bitcast i8* %7 to <32 x i8>*
  %wide.load14 = load <32 x i8>, <32 x i8>* %8, align 1
  %9 = sext <32 x i8> %wide.load14 to <32 x i32>
  %10 = mul nsw <32 x i32> %9, %6
  %11 = add nsw <32 x i32> %10, %vec.phi
  %index.next = add i64 %index, 32
  %12 = icmp eq i64 %index.next, %3
  br i1 %12, label %middle.block, label %vector.body

middle.block:
  %rdx.shuf1 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx1 = add <32 x i32> %11, %rdx.shuf1
  %rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf
  %rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx32 = add <32 x i32> %bin.rdx, %rdx.shuf15
  %rdx.shuf17 = shufflevector <32 x i32> %bin.rdx32, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx18 = add <32 x i32> %bin.rdx32, %rdx.shuf17
  %rdx.shuf19 = shufflevector <32 x i32> %bin.rdx18, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx20 = add <32 x i32> %bin.rdx18, %rdx.shuf19
  %13 = extractelement <32 x i32> %bin.rdx20, i32 0
  ret i32 %13
}

@a = dso_local global [1024 x i8] zeroinitializer, align 16
@b = dso_local global [1024 x i8] zeroinitializer, align 16

define dso_local i32 @sad_16i8_256() "min-legal-vector-width"="256" {
; CHECK-LABEL: sad_16i8_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT:    movq $-1024, %rax # imm = 0xFC00
; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB10_1: # %vector.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vmovdqu a+1024(%rax), %xmm2
; CHECK-NEXT:    vpsadbw b+1024(%rax), %xmm2, %xmm2
; CHECK-NEXT:    vpaddd %ymm1, %ymm2, %ymm1
; CHECK-NEXT:    addq $4, %rax
; CHECK-NEXT:    jne .LBB10_1
; CHECK-NEXT:  # %bb.2: # %middle.block
; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT:    vmovd %xmm0, %eax
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
entry:
  br label %vector.body

vector.body:
  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
  %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ]
  %0 = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %index
  %1 = bitcast i8* %0 to <16 x i8>*
  %wide.load = load <16 x i8>, <16 x i8>* %1, align 4
  %2 = zext <16 x i8> %wide.load to <16 x i32>
  %3 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %index
  %4 = bitcast i8* %3 to <16 x i8>*
  %wide.load1 = load <16 x i8>, <16 x i8>* %4, align 4
  %5 = zext <16 x i8> %wide.load1 to <16 x i32>
  %6 = sub nsw <16 x i32> %2, %5
  %7 = icmp sgt <16 x i32> %6, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
  %8 = sub nsw <16 x i32> zeroinitializer, %6
  %9 = select <16 x i1> %7, <16 x i32> %6, <16 x i32> %8
  %10 = add nsw <16 x i32> %9, %vec.phi
  %index.next = add i64 %index, 4
  %11 = icmp eq i64 %index.next, 1024
  br i1 %11, label %middle.block, label %vector.body

middle.block:
  %rdx.shuf = shufflevector <16 x i32> %10, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx = add <16 x i32> %10, %rdx.shuf
  %rdx.shuf2 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx2 = add <16 x i32> %bin.rdx, %rdx.shuf2
  %rdx.shuf3 = shufflevector <16 x i32> %bin.rdx2, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx3 = add <16 x i32> %bin.rdx2, %rdx.shuf3
  %rdx.shuf4 = shufflevector <16 x i32> %bin.rdx3, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx4 = add <16 x i32> %bin.rdx3, %rdx.shuf4
  %12 = extractelement <16 x i32> %bin.rdx4, i32 0
  ret i32 %12
}

define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" {
; CHECK-LABEL: sad_16i8_512:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT:    movq $-1024, %rax # imm = 0xFC00
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB11_1: # %vector.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vmovdqu a+1024(%rax), %xmm1
; CHECK-NEXT:    vpsadbw b+1024(%rax), %xmm1, %xmm1
; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
; CHECK-NEXT:    addq $4, %rax
; CHECK-NEXT:    jne .LBB11_1
; CHECK-NEXT:  # %bb.2: # %middle.block
; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
; CHECK-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT:    vmovd %xmm0, %eax
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
entry:
  br label %vector.body

vector.body:
  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
  %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ]
  %0 = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %index
  %1 = bitcast i8* %0 to <16 x i8>*
  %wide.load = load <16 x i8>, <16 x i8>* %1, align 4
  %2 = zext <16 x i8> %wide.load to <16 x i32>
  %3 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %index
  %4 = bitcast i8* %3 to <16 x i8>*
  %wide.load1 = load <16 x i8>, <16 x i8>* %4, align 4
  %5 = zext <16 x i8> %wide.load1 to <16 x i32>
  %6 = sub nsw <16 x i32> %2, %5
  %7 = icmp sgt <16 x i32> %6, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
  %8 = sub nsw <16 x i32> zeroinitializer, %6
  %9 = select <16 x i1> %7, <16 x i32> %6, <16 x i32> %8
  %10 = add nsw <16 x i32> %9, %vec.phi
  %index.next = add i64 %index, 4
  %11 = icmp eq i64 %index.next, 1024
  br i1 %11, label %middle.block, label %vector.body

middle.block:
  %rdx.shuf = shufflevector <16 x i32> %10, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx = add <16 x i32> %10, %rdx.shuf
  %rdx.shuf2 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx2 = add <16 x i32> %bin.rdx, %rdx.shuf2
  %rdx.shuf3 = shufflevector <16 x i32> %bin.rdx2, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx3 = add <16 x i32> %bin.rdx2, %rdx.shuf3
  %rdx.shuf4 = shufflevector <16 x i32> %bin.rdx3, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %bin.rdx4 = add <16 x i32> %bin.rdx3, %rdx.shuf4
  %12 = extractelement <16 x i32> %bin.rdx4, i32 0
  ret i32 %12
}

define dso_local void @sbto16f32_256(<16 x i16> %a, <16 x float>* %res) "min-legal-vector-width"="256" {
; CHECK-LABEL: sbto16f32_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    kshiftrw $8, %k0, %k1
; CHECK-NEXT:    vpmovm2d %k1, %ymm0
; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm0
; CHECK-NEXT:    vpmovm2d %k0, %ymm1
; CHECK-NEXT:    vcvtdq2ps %ymm1, %ymm1
; CHECK-NEXT:    vmovaps %ymm1, (%rdi)
; CHECK-NEXT:    vmovaps %ymm0, 32(%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = sitofp <16 x i1> %mask to <16 x float>
  store <16 x float> %1, <16 x float>* %res
  ret void
}

define dso_local void @sbto16f32_512(<16 x i16> %a, <16 x float>* %res) "min-legal-vector-width"="512" {
; CHECK-LABEL: sbto16f32_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    vpmovm2d %k0, %zmm0
; CHECK-NEXT:    vcvtdq2ps %zmm0, %zmm0
; CHECK-NEXT:    vmovaps %zmm0, (%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = sitofp <16 x i1> %mask to <16 x float>
  store <16 x float> %1, <16 x float>* %res
  ret void
}

define dso_local void @sbto16f64_256(<16 x i16> %a, <16 x double>* %res)  "min-legal-vector-width"="256" {
; CHECK-LABEL: sbto16f64_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    kshiftrw $8, %k0, %k1
; CHECK-NEXT:    vpmovm2d %k1, %ymm0
; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm1
; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm0
; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm0
; CHECK-NEXT:    vpmovm2d %k0, %ymm2
; CHECK-NEXT:    vcvtdq2pd %xmm2, %ymm3
; CHECK-NEXT:    vextracti128 $1, %ymm2, %xmm2
; CHECK-NEXT:    vcvtdq2pd %xmm2, %ymm2
; CHECK-NEXT:    vmovaps %ymm2, 32(%rdi)
; CHECK-NEXT:    vmovaps %ymm3, (%rdi)
; CHECK-NEXT:    vmovaps %ymm0, 96(%rdi)
; CHECK-NEXT:    vmovaps %ymm1, 64(%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = sitofp <16 x i1> %mask to <16 x double>
  store <16 x double> %1, <16 x double>* %res
  ret void
}

define dso_local void @sbto16f64_512(<16 x i16> %a, <16 x double>* %res)  "min-legal-vector-width"="512" {
; CHECK-LABEL: sbto16f64_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    vpmovm2d %k0, %zmm0
; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm1
; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm0
; CHECK-NEXT:    vmovaps %zmm0, 64(%rdi)
; CHECK-NEXT:    vmovaps %zmm1, (%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = sitofp <16 x i1> %mask to <16 x double>
  store <16 x double> %1, <16 x double>* %res
  ret void
}

define dso_local void @ubto16f32_256(<16 x i16> %a, <16 x float>* %res) "min-legal-vector-width"="256" {
; CHECK-LABEL: ubto16f32_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    kshiftrw $8, %k0, %k1
; CHECK-NEXT:    vpmovm2d %k1, %ymm0
; CHECK-NEXT:    vpsrld $31, %ymm0, %ymm0
; CHECK-NEXT:    vcvtdq2ps %ymm0, %ymm0
; CHECK-NEXT:    vpmovm2d %k0, %ymm1
; CHECK-NEXT:    vpsrld $31, %ymm1, %ymm1
; CHECK-NEXT:    vcvtdq2ps %ymm1, %ymm1
; CHECK-NEXT:    vmovaps %ymm1, (%rdi)
; CHECK-NEXT:    vmovaps %ymm0, 32(%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = uitofp <16 x i1> %mask to <16 x float>
  store <16 x float> %1, <16 x float>* %res
  ret void
}

define dso_local void @ubto16f32_512(<16 x i16> %a, <16 x float>* %res) "min-legal-vector-width"="512" {
; CHECK-LABEL: ubto16f32_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    vpmovm2d %k0, %zmm0
; CHECK-NEXT:    vpsrld $31, %zmm0, %zmm0
; CHECK-NEXT:    vcvtdq2ps %zmm0, %zmm0
; CHECK-NEXT:    vmovaps %zmm0, (%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = uitofp <16 x i1> %mask to <16 x float>
  store <16 x float> %1, <16 x float>* %res
  ret void
}

define dso_local void @ubto16f64_256(<16 x i16> %a, <16 x double>* %res) "min-legal-vector-width"="256" {
; CHECK-LABEL: ubto16f64_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    kshiftrw $8, %k0, %k1
; CHECK-NEXT:    vpmovm2d %k1, %ymm0
; CHECK-NEXT:    vpsrld $31, %ymm0, %ymm0
; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm1
; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm0
; CHECK-NEXT:    vcvtdq2pd %xmm0, %ymm0
; CHECK-NEXT:    vpmovm2d %k0, %ymm2
; CHECK-NEXT:    vpsrld $31, %ymm2, %ymm2
; CHECK-NEXT:    vcvtdq2pd %xmm2, %ymm3
; CHECK-NEXT:    vextracti128 $1, %ymm2, %xmm2
; CHECK-NEXT:    vcvtdq2pd %xmm2, %ymm2
; CHECK-NEXT:    vmovaps %ymm2, 32(%rdi)
; CHECK-NEXT:    vmovaps %ymm3, (%rdi)
; CHECK-NEXT:    vmovaps %ymm0, 96(%rdi)
; CHECK-NEXT:    vmovaps %ymm1, 64(%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = uitofp <16 x i1> %mask to <16 x double>
  store <16 x double> %1, <16 x double>* %res
  ret void
}

define dso_local void @ubto16f64_512(<16 x i16> %a, <16 x double>* %res) "min-legal-vector-width"="512" {
; CHECK-LABEL: ubto16f64_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovw2m %ymm0, %k0
; CHECK-NEXT:    vpmovm2d %k0, %zmm0
; CHECK-NEXT:    vpsrld $31, %zmm0, %zmm0
; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm1
; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
; CHECK-NEXT:    vcvtdq2pd %ymm0, %zmm0
; CHECK-NEXT:    vmovaps %zmm0, 64(%rdi)
; CHECK-NEXT:    vmovaps %zmm1, (%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %mask = icmp slt <16 x i16> %a, zeroinitializer
  %1 = uitofp <16 x i1> %mask to <16 x double>
  store <16 x double> %1, <16 x double>* %res
  ret void
}

define <16 x i16> @test_16f32toub_256(<16 x float>* %ptr, <16 x i16> %passthru) "min-legal-vector-width"="256" {
; CHECK-LABEL: test_16f32toub_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vcvttps2dq (%rdi), %ymm1
; CHECK-NEXT:    vpslld $31, %ymm1, %ymm1
; CHECK-NEXT:    vpmovd2m %ymm1, %k0
; CHECK-NEXT:    vcvttps2dq 32(%rdi), %ymm1
; CHECK-NEXT:    vpslld $31, %ymm1, %ymm1
; CHECK-NEXT:    vpmovd2m %ymm1, %k1
; CHECK-NEXT:    kunpckbw %k0, %k1, %k1
; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm0 {%k1} {z}
; CHECK-NEXT:    retq
  %a = load <16 x float>, <16 x float>* %ptr
  %mask = fptoui <16 x float> %a to <16 x i1>
  %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer
  ret <16 x i16> %select
}

define <16 x i16> @test_16f32toub_512(<16 x float>* %ptr, <16 x i16> %passthru) "min-legal-vector-width"="512" {
; CHECK-LABEL: test_16f32toub_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vcvttps2dq (%rdi), %zmm1
; CHECK-NEXT:    vpslld $31, %zmm1, %zmm1
; CHECK-NEXT:    vpmovd2m %zmm1, %k1
; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm0 {%k1} {z}
; CHECK-NEXT:    retq
  %a = load <16 x float>, <16 x float>* %ptr
  %mask = fptoui <16 x float> %a to <16 x i1>
  %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer
  ret <16 x i16> %select
}

define <16 x i16> @test_16f32tosb_256(<16 x float>* %ptr, <16 x i16> %passthru) "min-legal-vector-width"="256" {
; CHECK-LABEL: test_16f32tosb_256:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vcvttps2dq (%rdi), %ymm1
; CHECK-NEXT:    vpmovd2m %ymm1, %k0
; CHECK-NEXT:    vcvttps2dq 32(%rdi), %ymm1
; CHECK-NEXT:    vpmovd2m %ymm1, %k1
; CHECK-NEXT:    kunpckbw %k0, %k1, %k1
; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm0 {%k1} {z}
; CHECK-NEXT:    retq
  %a = load <16 x float>, <16 x float>* %ptr
  %mask = fptosi <16 x float> %a to <16 x i1>
  %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer
  ret <16 x i16> %select
}

define <16 x i16> @test_16f32tosb_512(<16 x float>* %ptr, <16 x i16> %passthru) "min-legal-vector-width"="512" {
; CHECK-LABEL: test_16f32tosb_512:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vcvttps2dq (%rdi), %zmm1
; CHECK-NEXT:    vpmovd2m %zmm1, %k1
; CHECK-NEXT:    vmovdqu16 %ymm0, %ymm0 {%k1} {z}
; CHECK-NEXT:    retq
  %a = load <16 x float>, <16 x float>* %ptr
  %mask = fptosi <16 x float> %a to <16 x i1>
  %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer
  ret <16 x i16> %select
}

define dso_local void @mul256(<64 x i8>* %a, <64 x i8>* %b, <64 x i8>* %c) "min-legal-vector-width"="256" {
; CHECK-AVX512-LABEL: mul256:
; CHECK-AVX512:       # %bb.0:
; CHECK-AVX512-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-AVX512-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-AVX512-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-AVX512-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-AVX512-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
; CHECK-AVX512-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; CHECK-AVX512-NEXT:    vpand %ymm5, %ymm4, %ymm4
; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-AVX512-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
; CHECK-AVX512-NEXT:    vpand %ymm5, %ymm1, %ymm1
; CHECK-AVX512-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-AVX512-NEXT:    vpmullw %ymm3, %ymm4, %ymm3
; CHECK-AVX512-NEXT:    vpand %ymm5, %ymm3, %ymm3
; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-AVX512-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
; CHECK-AVX512-NEXT:    vpand %ymm5, %ymm0, %ymm0
; CHECK-AVX512-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
; CHECK-AVX512-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-AVX512-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-AVX512-NEXT:    vzeroupper
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: mul256:
; CHECK-VBMI:       # %bb.0:
; CHECK-VBMI-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-VBMI-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-VBMI-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-VBMI-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-VBMI-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-VBMI-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,2,4,6,8,10,12,14,32,34,36,38,40,42,44,46,16,18,20,22,24,26,28,30,48,50,52,54,56,58,60,62]
; CHECK-VBMI-NEXT:    vpermt2b %ymm4, %ymm3, %ymm1
; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-VBMI-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
; CHECK-VBMI-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
; CHECK-VBMI-NEXT:    vpermt2b %ymm4, %ymm3, %ymm0
; CHECK-VBMI-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-VBMI-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-VBMI-NEXT:    vzeroupper
; CHECK-VBMI-NEXT:    retq
  %d = load <64 x i8>, <64 x i8>* %a
  %e = load <64 x i8>, <64 x i8>* %b
  %f = mul <64 x i8> %d, %e
  store <64 x i8> %f, <64 x i8>* %c
  ret void
}

define dso_local void @mul512(<64 x i8>* %a, <64 x i8>* %b, <64 x i8>* %c) "min-legal-vector-width"="512" {
; CHECK-AVX512-LABEL: mul512:
; CHECK-AVX512:       # %bb.0:
; CHECK-AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
; CHECK-AVX512-NEXT:    vmovdqa64 (%rsi), %zmm1
; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; CHECK-AVX512-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
; CHECK-AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
; CHECK-AVX512-NEXT:    vpandq %zmm3, %zmm2, %zmm2
; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
; CHECK-AVX512-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
; CHECK-AVX512-NEXT:    vpandq %zmm3, %zmm0, %zmm0
; CHECK-AVX512-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
; CHECK-AVX512-NEXT:    vmovdqa64 %zmm0, (%rdx)
; CHECK-AVX512-NEXT:    vzeroupper
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: mul512:
; CHECK-VBMI:       # %bb.0:
; CHECK-VBMI-NEXT:    vmovdqa64 (%rdi), %zmm0
; CHECK-VBMI-NEXT:    vmovdqa64 (%rsi), %zmm1
; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; CHECK-VBMI-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
; CHECK-VBMI-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
; CHECK-VBMI-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,2,4,6,8,10,12,14,64,66,68,70,72,74,76,78,16,18,20,22,24,26,28,30,80,82,84,86,88,90,92,94,32,34,36,38,40,42,44,46,96,98,100,102,104,106,108,110,48,50,52,54,56,58,60,62,112,114,116,118,120,122,124,126]
; CHECK-VBMI-NEXT:    vpermi2b %zmm2, %zmm0, %zmm1
; CHECK-VBMI-NEXT:    vmovdqa64 %zmm1, (%rdx)
; CHECK-VBMI-NEXT:    vzeroupper
; CHECK-VBMI-NEXT:    retq
  %d = load <64 x i8>, <64 x i8>* %a
  %e = load <64 x i8>, <64 x i8>* %b
  %f = mul <64 x i8> %d, %e
  store <64 x i8> %f, <64 x i8>* %c
  ret void
}

; This threw an assertion at one point.
define <4 x i32> @mload_v4i32(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %dst) "min-legal-vector-width"="256" {
; CHECK-LABEL: mload_v4i32:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vptestnmd %xmm0, %xmm0, %k1
; CHECK-NEXT:    vpblendmd (%rdi), %xmm1, %xmm0 {%k1}
; CHECK-NEXT:    retq
  %mask = icmp eq <4 x i32> %trigger, zeroinitializer
  %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
  ret <4 x i32> %res
}
declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32, <4 x i1>, <4 x i32>)

define <16 x i32> @trunc_v16i64_v16i32(<16 x i64>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i64_v16i32:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vmovdqa 64(%rdi), %ymm2
; CHECK-NEXT:    vmovdqa 96(%rdi), %ymm3
; CHECK-NEXT:    vpmovqd %ymm0, %xmm0
; CHECK-NEXT:    vpmovqd %ymm1, %xmm1
; CHECK-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
; CHECK-NEXT:    vpmovqd %ymm2, %xmm1
; CHECK-NEXT:    vpmovqd %ymm3, %xmm2
; CHECK-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
; CHECK-NEXT:    retq
  %a = load <16 x i64>, <16 x i64>* %x
  %b = trunc <16 x i64> %a to <16 x i32>
  ret <16 x i32> %b
}

define <16 x i8> @trunc_v16i64_v16i8(<16 x i64>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i64_v16i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vmovdqa 64(%rdi), %ymm2
; CHECK-NEXT:    vmovdqa 96(%rdi), %ymm3
; CHECK-NEXT:    vpmovqb %ymm3, %xmm3
; CHECK-NEXT:    vpmovqb %ymm2, %xmm2
; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; CHECK-NEXT:    vpmovqb %ymm1, %xmm1
; CHECK-NEXT:    vpmovqb %ymm0, %xmm0
; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = load <16 x i64>, <16 x i64>* %x
  %b = trunc <16 x i64> %a to <16 x i8>
  ret <16 x i8> %b
}

define <16 x i8> @trunc_v16i32_v16i8(<16 x i32>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i32_v16i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vpmovdb %ymm1, %xmm1
; CHECK-NEXT:    vpmovdb %ymm0, %xmm0
; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = load <16 x i32>, <16 x i32>* %x
  %b = trunc <16 x i32> %a to <16 x i8>
  ret <16 x i8> %b
}

define <8 x i8> @trunc_v8i64_v8i8(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v8i64_v8i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vpmovqb %ymm1, %xmm1
; CHECK-NEXT:    vpmovqb %ymm0, %xmm0
; CHECK-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = load <8 x i64>, <8 x i64>* %x
  %b = trunc <8 x i64> %a to <8 x i8>
  ret <8 x i8> %b
}

define <8 x i16> @trunc_v8i64_v8i16(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v8i64_v8i16:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm1
; CHECK-NEXT:    vpmovqw %ymm1, %xmm1
; CHECK-NEXT:    vpmovqw %ymm0, %xmm0
; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = load <8 x i64>, <8 x i64>* %x
  %b = trunc <8 x i64> %a to <8 x i16>
  ret <8 x i16> %b
}

define <8 x i32> @trunc_v8i64_v8i32_zeroes(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v8i64_v8i32_zeroes:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpsrlq $48, 32(%rdi), %ymm0
; CHECK-NEXT:    vpsrlq $48, (%rdi), %ymm1
; CHECK-NEXT:    vpackusdw %ymm0, %ymm1, %ymm0
; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-NEXT:    retq
  %a = load <8 x i64>, <8 x i64>* %x
  %b = lshr <8 x i64> %a, <i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48>
  %c = trunc <8 x i64> %b to <8 x i32>
  ret <8 x i32> %c
}

define <16 x i16> @trunc_v16i32_v16i16_zeroes(<16 x i32>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i32_v16i16_zeroes:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm1
; CHECK-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
; CHECK-NEXT:    vpermi2w 32(%rdi), %ymm1, %ymm0
; CHECK-NEXT:    retq
  %a = load <16 x i32>, <16 x i32>* %x
  %b = lshr <16 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %c = trunc <16 x i32> %b to <16 x i16>
  ret <16 x i16> %c
}

define <32 x i8> @trunc_v32i16_v32i8_zeroes(<32 x i16>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-AVX512-LABEL: trunc_v32i16_v32i8_zeroes:
; CHECK-AVX512:       # %bb.0:
; CHECK-AVX512-NEXT:    vpsrlw $8, 32(%rdi), %ymm0
; CHECK-AVX512-NEXT:    vpsrlw $8, (%rdi), %ymm1
; CHECK-AVX512-NEXT:    vpackuswb %ymm0, %ymm1, %ymm0
; CHECK-AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: trunc_v32i16_v32i8_zeroes:
; CHECK-VBMI:       # %bb.0:
; CHECK-VBMI-NEXT:    vmovdqa (%rdi), %ymm1
; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
; CHECK-VBMI-NEXT:    vpermi2b 32(%rdi), %ymm1, %ymm0
; CHECK-VBMI-NEXT:    retq
  %a = load <32 x i16>, <32 x i16>* %x
  %b = lshr <32 x i16> %a, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %c = trunc <32 x i16> %b to <32 x i8>
  ret <32 x i8> %c
}

define <8 x i32> @trunc_v8i64_v8i32_sign(<8 x i64>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v8i64_v8i32_sign:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpsraq $48, 32(%rdi), %ymm0
; CHECK-NEXT:    vpsraq $48, (%rdi), %ymm1
; CHECK-NEXT:    vpmovqd %ymm1, %xmm1
; CHECK-NEXT:    vpmovqd %ymm0, %xmm0
; CHECK-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
; CHECK-NEXT:    retq
  %a = load <8 x i64>, <8 x i64>* %x
  %b = ashr <8 x i64> %a, <i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48>
  %c = trunc <8 x i64> %b to <8 x i32>
  ret <8 x i32> %c
}

define <16 x i16> @trunc_v16i32_v16i16_sign(<16 x i32>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_v16i32_v16i16_sign:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm1
; CHECK-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]
; CHECK-NEXT:    vpermi2w 32(%rdi), %ymm1, %ymm0
; CHECK-NEXT:    retq
  %a = load <16 x i32>, <16 x i32>* %x
  %b = ashr <16 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
  %c = trunc <16 x i32> %b to <16 x i16>
  ret <16 x i16> %c
}

define <32 x i8> @trunc_v32i16_v32i8_sign(<32 x i16>* %x) nounwind "min-legal-vector-width"="256" {
; CHECK-AVX512-LABEL: trunc_v32i16_v32i8_sign:
; CHECK-AVX512:       # %bb.0:
; CHECK-AVX512-NEXT:    vpsrlw $8, 32(%rdi), %ymm0
; CHECK-AVX512-NEXT:    vpsrlw $8, (%rdi), %ymm1
; CHECK-AVX512-NEXT:    vpackuswb %ymm0, %ymm1, %ymm0
; CHECK-AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: trunc_v32i16_v32i8_sign:
; CHECK-VBMI:       # %bb.0:
; CHECK-VBMI-NEXT:    vmovdqa (%rdi), %ymm1
; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]
; CHECK-VBMI-NEXT:    vpermi2b 32(%rdi), %ymm1, %ymm0
; CHECK-VBMI-NEXT:    retq
  %a = load <32 x i16>, <32 x i16>* %x
  %b = ashr <32 x i16> %a, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
  %c = trunc <32 x i16> %b to <32 x i8>
  ret <32 x i8> %c
}

define dso_local void @zext_v16i8_v16i64(<16 x i8> %x, <16 x i64>* %y) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: zext_v16i8_v16i64:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; CHECK-NEXT:    vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
; CHECK-NEXT:    vextracti128 $1, %ymm1, %xmm1
; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
; CHECK-NEXT:    vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
; CHECK-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
; CHECK-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
; CHECK-NEXT:    vmovdqa %ymm0, (%rdi)
; CHECK-NEXT:    vmovdqa %ymm1, 64(%rdi)
; CHECK-NEXT:    vmovdqa %ymm3, 96(%rdi)
; CHECK-NEXT:    vmovdqa %ymm2, 32(%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = zext <16 x i8> %x to <16 x i64>
  store <16 x i64> %a, <16 x i64>* %y
  ret void
}

define dso_local void @sext_v16i8_v16i64(<16 x i8> %x, <16 x i64>* %y) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: sext_v16i8_v16i64:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpmovsxbw %xmm0, %ymm1
; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; CHECK-NEXT:    vpmovsxwq %xmm2, %ymm2
; CHECK-NEXT:    vextracti128 $1, %ymm1, %xmm1
; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
; CHECK-NEXT:    vpmovsxwq %xmm3, %ymm3
; CHECK-NEXT:    vpmovsxwq %xmm1, %ymm1
; CHECK-NEXT:    vpmovsxbq %xmm0, %ymm0
; CHECK-NEXT:    vmovdqa %ymm0, (%rdi)
; CHECK-NEXT:    vmovdqa %ymm1, 64(%rdi)
; CHECK-NEXT:    vmovdqa %ymm3, 96(%rdi)
; CHECK-NEXT:    vmovdqa %ymm2, 32(%rdi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = sext <16 x i8> %x to <16 x i64>
  store <16 x i64> %a, <16 x i64>* %y
  ret void
}

define dso_local void @vselect_split_v8i16_setcc(<8 x i16> %s, <8 x i16> %t, <8 x i64>* %p, <8 x i64>* %q, <8 x i64>* %r) "min-legal-vector-width"="256" {
; CHECK-LABEL: vselect_split_v8i16_setcc:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-NEXT:    vpcmpeqw %xmm1, %xmm0, %k1
; CHECK-NEXT:    kshiftrb $4, %k1, %k2
; CHECK-NEXT:    vmovdqa64 32(%rdi), %ymm3 {%k2}
; CHECK-NEXT:    vmovdqa64 (%rdi), %ymm2 {%k1}
; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <8 x i64>, <8 x i64>* %p
  %y = load <8 x i64>, <8 x i64>* %q
  %a = icmp eq <8 x i16> %s, %t
  %b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y
  store <8 x i64> %b, <8 x i64>* %r
  ret void
}

define dso_local void @vselect_split_v8i32_setcc(<8 x i32> %s, <8 x i32> %t, <8 x i64>* %p, <8 x i64>* %q, <8 x i64>* %r) "min-legal-vector-width"="256" {
; CHECK-LABEL: vselect_split_v8i32_setcc:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-NEXT:    vpcmpeqd %ymm1, %ymm0, %k1
; CHECK-NEXT:    kshiftrb $4, %k1, %k2
; CHECK-NEXT:    vmovdqa64 32(%rdi), %ymm3 {%k2}
; CHECK-NEXT:    vmovdqa64 (%rdi), %ymm2 {%k1}
; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <8 x i64>, <8 x i64>* %p
  %y = load <8 x i64>, <8 x i64>* %q
  %a = icmp eq <8 x i32> %s, %t
  %b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y
  store <8 x i64> %b, <8 x i64>* %r
  ret void
}

define dso_local void @vselect_split_v16i8_setcc(<16 x i8> %s, <16 x i8> %t, <16 x i32>* %p, <16 x i32>* %q, <16 x i32>* %r) "min-legal-vector-width"="256" {
; CHECK-LABEL: vselect_split_v16i8_setcc:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-NEXT:    vpcmpeqb %xmm1, %xmm0, %k1
; CHECK-NEXT:    kshiftrw $8, %k1, %k2
; CHECK-NEXT:    vmovdqa32 32(%rdi), %ymm3 {%k2}
; CHECK-NEXT:    vmovdqa32 (%rdi), %ymm2 {%k1}
; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <16 x i32>, <16 x i32>* %p
  %y = load <16 x i32>, <16 x i32>* %q
  %a = icmp eq <16 x i8> %s, %t
  %b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y
  store <16 x i32> %b, <16 x i32>* %r
  ret void
}

define dso_local void @vselect_split_v16i16_setcc(<16 x i16> %s, <16 x i16> %t, <16 x i32>* %p, <16 x i32>* %q, <16 x i32>* %r) "min-legal-vector-width"="256" {
; CHECK-LABEL: vselect_split_v16i16_setcc:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
; CHECK-NEXT:    vpcmpeqw %ymm1, %ymm0, %k1
; CHECK-NEXT:    kshiftrw $8, %k1, %k2
; CHECK-NEXT:    vmovdqa32 32(%rdi), %ymm3 {%k2}
; CHECK-NEXT:    vmovdqa32 (%rdi), %ymm2 {%k1}
; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <16 x i32>, <16 x i32>* %p
  %y = load <16 x i32>, <16 x i32>* %q
  %a = icmp eq <16 x i16> %s, %t
  %b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y
  store <16 x i32> %b, <16 x i32>* %r
  ret void
}

define <16 x i8> @trunc_packus_v16i32_v16i8(<16 x i32>* %p) "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_packus_v16i32_v16i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vpackusdw 32(%rdi), %ymm0, %ymm0
; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-NEXT:    vpmovuswb %ymm0, %xmm0
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = load <16 x i32>, <16 x i32>* %p
  %b = icmp slt <16 x i32> %a, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
  %c = select <16 x i1> %b, <16 x i32> %a, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
  %d = icmp sgt <16 x i32> %c, zeroinitializer
  %e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> zeroinitializer
  %f = trunc <16 x i32> %e to <16 x i8>
  ret <16 x i8> %f
}

define dso_local void @trunc_packus_v16i32_v16i8_store(<16 x i32>* %p, <16 x i8>* %q) "min-legal-vector-width"="256" {
; CHECK-LABEL: trunc_packus_v16i32_v16i8_store:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rdi), %ymm0
; CHECK-NEXT:    vpackusdw 32(%rdi), %ymm0, %ymm0
; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; CHECK-NEXT:    vpmovuswb %ymm0, (%rsi)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %a = load <16 x i32>, <16 x i32>* %p
  %b = icmp slt <16 x i32> %a, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
  %c = select <16 x i1> %b, <16 x i32> %a, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
  %d = icmp sgt <16 x i32> %c, zeroinitializer
  %e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> zeroinitializer
  %f = trunc <16 x i32> %e to <16 x i8>
  store <16 x i8> %f, <16 x i8>* %q
  ret void
}

define <64 x i1> @v64i1_argument_return(<64 x i1> %x) "min-legal-vector-width"="256" {
; CHECK-LABEL: v64i1_argument_return:
; CHECK:       # %bb.0:
; CHECK-NEXT:    retq
  ret <64 x i1> %x
}

define dso_local void @v64i1_shuffle(<64 x i8>* %x, <64 x i8>* %y) "min-legal-vector-width"="256" {
; CHECK-LABEL: v64i1_shuffle:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    vmovdqa (%rdi), %ymm1
; CHECK-NEXT:    vmovdqa 32(%rdi), %ymm0
; CHECK-NEXT:    vptestnmb %ymm1, %ymm1, %k0
; CHECK-NEXT:    kshiftrd $1, %k0, %k1
; CHECK-NEXT:    movq $-3, %rax
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftlq $63, %k0, %k2
; CHECK-NEXT:    kshiftrq $62, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-5, %rax
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $3, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $61, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-9, %rax
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $2, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $60, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-17, %rax
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $5, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $59, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-33, %rax
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $4, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $58, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-65, %rax
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $7, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $57, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-129, %rax
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $6, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $56, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-257, %rax # imm = 0xFEFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $9, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $55, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-513, %rax # imm = 0xFDFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $8, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $54, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-1025, %rax # imm = 0xFBFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $11, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $53, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-2049, %rax # imm = 0xF7FF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $10, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $52, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-4097, %rax # imm = 0xEFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $13, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $51, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-8193, %rax # imm = 0xDFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $12, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $50, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-16385, %rax # imm = 0xBFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $15, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $49, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-32769, %rax # imm = 0xFFFF7FFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $14, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $48, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-65537, %rax # imm = 0xFFFEFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $17, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $47, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-131073, %rax # imm = 0xFFFDFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $16, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $46, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-262145, %rax # imm = 0xFFFBFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $19, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $45, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-524289, %rax # imm = 0xFFF7FFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $18, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $44, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-1048577, %rax # imm = 0xFFEFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $21, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $43, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-2097153, %rax # imm = 0xFFDFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $20, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $42, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-4194305, %rax # imm = 0xFFBFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $23, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $41, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-8388609, %rax # imm = 0xFF7FFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $22, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $40, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-16777217, %rax # imm = 0xFEFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $25, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $39, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-33554433, %rax # imm = 0xFDFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $24, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $38, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-67108865, %rax # imm = 0xFBFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $27, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $37, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-134217729, %rax # imm = 0xF7FFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $26, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $36, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-268435457, %rax # imm = 0xEFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $29, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $35, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-536870913, %rax # imm = 0xDFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $28, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $34, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movq $-1073741825, %rax # imm = 0xBFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k1
; CHECK-NEXT:    kshiftrd $31, %k0, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $33, %k2, %k2
; CHECK-NEXT:    korq %k2, %k1, %k1
; CHECK-NEXT:    movabsq $-2147483649, %rax # imm = 0xFFFFFFFF7FFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k1, %k2
; CHECK-NEXT:    vptestnmb %ymm0, %ymm0, %k1
; CHECK-NEXT:    kshiftrd $30, %k0, %k0
; CHECK-NEXT:    kshiftlq $63, %k0, %k0
; CHECK-NEXT:    kshiftrq $32, %k0, %k0
; CHECK-NEXT:    korq %k0, %k2, %k0
; CHECK-NEXT:    movabsq $-4294967297, %rax # imm = 0xFFFFFFFEFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $1, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $31, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-8589934593, %rax # imm = 0xFFFFFFFDFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftlq $63, %k1, %k2
; CHECK-NEXT:    kshiftrq $30, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-17179869185, %rax # imm = 0xFFFFFFFBFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $3, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $29, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-34359738369, %rax # imm = 0xFFFFFFF7FFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $2, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $28, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-68719476737, %rax # imm = 0xFFFFFFEFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $5, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $27, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-137438953473, %rax # imm = 0xFFFFFFDFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $4, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $26, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-274877906945, %rax # imm = 0xFFFFFFBFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $7, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $25, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-549755813889, %rax # imm = 0xFFFFFF7FFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $6, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $24, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-1099511627777, %rax # imm = 0xFFFFFEFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $9, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $23, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-2199023255553, %rax # imm = 0xFFFFFDFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $8, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $22, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-4398046511105, %rax # imm = 0xFFFFFBFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $11, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $21, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-8796093022209, %rax # imm = 0xFFFFF7FFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $10, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $20, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-17592186044417, %rax # imm = 0xFFFFEFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $13, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $19, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-35184372088833, %rax # imm = 0xFFFFDFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $12, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $18, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-70368744177665, %rax # imm = 0xFFFFBFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $15, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $17, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-140737488355329, %rax # imm = 0xFFFF7FFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $14, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $16, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-281474976710657, %rax # imm = 0xFFFEFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $17, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $15, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-562949953421313, %rax # imm = 0xFFFDFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $16, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $14, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-1125899906842625, %rax # imm = 0xFFFBFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $19, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $13, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-2251799813685249, %rax # imm = 0xFFF7FFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $18, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $12, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-4503599627370497, %rax # imm = 0xFFEFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $21, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $11, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-9007199254740993, %rax # imm = 0xFFDFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $20, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $10, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-18014398509481985, %rax # imm = 0xFFBFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $23, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $9, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-36028797018963969, %rax # imm = 0xFF7FFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $22, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $8, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-72057594037927937, %rax # imm = 0xFEFFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $25, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $7, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-144115188075855873, %rax # imm = 0xFDFFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $24, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $6, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-288230376151711745, %rax # imm = 0xFBFFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $27, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $5, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-576460752303423489, %rax # imm = 0xF7FFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $26, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $4, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-1152921504606846977, %rax # imm = 0xEFFFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $29, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $3, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-2305843009213693953, %rax # imm = 0xDFFFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $28, %k1, %k2
; CHECK-NEXT:    kshiftlq $63, %k2, %k2
; CHECK-NEXT:    kshiftrq $2, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    movabsq $-4611686018427387905, %rax # imm = 0xBFFFFFFFFFFFFFFF
; CHECK-NEXT:    kmovq %rax, %k2
; CHECK-NEXT:    kandq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $31, %k1, %k2
; CHECK-NEXT:    kshiftlq $62, %k2, %k2
; CHECK-NEXT:    korq %k2, %k0, %k0
; CHECK-NEXT:    kshiftrd $30, %k1, %k1
; CHECK-NEXT:    kshiftlq $1, %k0, %k0
; CHECK-NEXT:    kshiftrq $1, %k0, %k0
; CHECK-NEXT:    kshiftlq $63, %k1, %k1
; CHECK-NEXT:    korq %k1, %k0, %k1
; CHECK-NEXT:    vmovdqu8 %ymm1, (%rsi) {%k1}
; CHECK-NEXT:    kshiftrq $32, %k1, %k1
; CHECK-NEXT:    vmovdqu8 %ymm0, 32(%rsi) {%k1}
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
entry:
  %a = load <64 x i8>, <64 x i8>* %x
  %b = icmp eq <64 x i8> %a, zeroinitializer
  %shuf = shufflevector <64 x i1> %b, <64 x i1> undef, <64 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30, i32 33, i32 32, i32 35, i32 34, i32 37, i32 36, i32 39, i32 38, i32 41, i32 40, i32 43, i32 42, i32 45, i32 44, i32 47, i32 46, i32 49, i32 48, i32 51, i32 50, i32 53, i32 52, i32 55, i32 54, i32 57, i32 56, i32 59, i32 58, i32 61, i32 60, i32 63, i32 62>
  call void @llvm.masked.store.v64i8.p0v64i8(<64 x i8> %a, <64 x i8>* %y, i32 1, <64 x i1> %shuf)
  ret void
}
declare void @llvm.masked.store.v64i8.p0v64i8(<64 x i8>, <64 x i8>*, i32, <64 x i1>)

@mem64_dst = dso_local global i64 0, align 8
@mem64_src = dso_local global i64 0, align 8
define dso_local i32 @v64i1_inline_asm() "min-legal-vector-width"="256" {
; CHECK-LABEL: v64i1_inline_asm:
; CHECK:       # %bb.0:
; CHECK-NEXT:    kmovq {{.*}}(%rip), %k0
; CHECK-NEXT:    #APP
; CHECK-NEXT:    #NO_APP
; CHECK-NEXT:    kmovq %k0, {{.*}}(%rip)
; CHECK-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
; CHECK-NEXT:    retq
  %1 = alloca i32, align 4
  %2 = load i64, i64* @mem64_src, align 8
  %3 = call i64 asm "", "=k,k,~{dirflag},~{fpsr},~{flags}"(i64 %2)
  store i64 %3, i64* @mem64_dst, align 8
  %4 = load i32, i32* %1, align 4
  ret i32 %4
}

define dso_local void @cmp_v8i64_sext(<8 x i64>* %xptr, <8 x i64>* %yptr, <8 x i64>* %zptr) "min-legal-vector-width"="256" {
; CHECK-LABEL: cmp_v8i64_sext:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rsi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm1
; CHECK-NEXT:    vpcmpgtq 32(%rdi), %ymm1, %ymm1
; CHECK-NEXT:    vpcmpgtq (%rdi), %ymm0, %ymm0
; CHECK-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <8 x i64>, <8 x i64>* %xptr
  %y = load <8 x i64>, <8 x i64>* %yptr
  %cmp = icmp slt <8 x i64> %x, %y
  %ext = sext <8 x i1> %cmp to <8 x i64>
  store <8 x i64> %ext, <8 x i64>* %zptr
  ret void
}

define dso_local void @cmp_v8i64_zext(<8 x i64>* %xptr, <8 x i64>* %yptr, <8 x i64>* %zptr) "min-legal-vector-width"="256" {
; CHECK-LABEL: cmp_v8i64_zext:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa (%rsi), %ymm0
; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm1
; CHECK-NEXT:    vpcmpgtq 32(%rdi), %ymm1, %ymm1
; CHECK-NEXT:    vpcmpgtq (%rdi), %ymm0, %ymm0
; CHECK-NEXT:    vpsrlq $63, %ymm1, %ymm1
; CHECK-NEXT:    vpsrlq $63, %ymm0, %ymm0
; CHECK-NEXT:    vmovdqa %ymm0, (%rdx)
; CHECK-NEXT:    vmovdqa %ymm1, 32(%rdx)
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %x = load <8 x i64>, <8 x i64>* %xptr
  %y = load <8 x i64>, <8 x i64>* %yptr
  %cmp = icmp slt <8 x i64> %x, %y
  %ext = zext <8 x i1> %cmp to <8 x i64>
  store <8 x i64> %ext, <8 x i64>* %zptr
  ret void
}

define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: var_rotate_v16i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; CHECK-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
; CHECK-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
; CHECK-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
; CHECK-NEXT:    vpsllvw %ymm1, %ymm0, %ymm1
; CHECK-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
; CHECK-NEXT:    vpsrlvw %ymm2, %ymm0, %ymm0
; CHECK-NEXT:    vpor %ymm0, %ymm1, %ymm0
; CHECK-NEXT:    vpmovwb %ymm0, %xmm0
; CHECK-NEXT:    vzeroupper
; CHECK-NEXT:    retq
  %b8 = sub <16 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %b
  %shl = shl <16 x i8> %a, %b
  %lshr = lshr <16 x i8> %a, %b8
  %or = or <16 x i8> %shl, %lshr
  ret <16 x i8> %or
}

define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: var_rotate_v32i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpsllw $4, %ymm0, %ymm2
; CHECK-NEXT:    vpsrlw $4, %ymm0, %ymm3
; CHECK-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
; CHECK-NEXT:    vpsllw $5, %ymm1, %ymm1
; CHECK-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
; CHECK-NEXT:    vpsllw $2, %ymm0, %ymm2
; CHECK-NEXT:    vpsrlw $6, %ymm0, %ymm3
; CHECK-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
; CHECK-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
; CHECK-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
; CHECK-NEXT:    vpsrlw $7, %ymm0, %ymm2
; CHECK-NEXT:    vpaddb %ymm0, %ymm0, %ymm3
; CHECK-NEXT:    vpternlogq $248, {{.*}}(%rip), %ymm2, %ymm3
; CHECK-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
; CHECK-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
; CHECK-NEXT:    retq
  %b8 = sub <32 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %b
  %shl = shl <32 x i8> %a, %b
  %lshr = lshr <32 x i8> %a, %b8
  %or = or <32 x i8> %shl, %lshr
  ret <32 x i8> %or
}

define <32 x i8> @splatvar_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" {
; CHECK-AVX512-LABEL: splatvar_rotate_v32i8:
; CHECK-AVX512:       # %bb.0:
; CHECK-AVX512-NEXT:    vpbroadcastb %xmm1, %xmm1
; CHECK-AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
; CHECK-AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
; CHECK-AVX512-NEXT:    vpsllw %xmm2, %ymm0, %ymm3
; CHECK-AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; CHECK-AVX512-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
; CHECK-AVX512-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
; CHECK-AVX512-NEXT:    vpsllw %xmm2, %xmm4, %xmm2
; CHECK-AVX512-NEXT:    vpbroadcastb %xmm2, %ymm2
; CHECK-AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
; CHECK-AVX512-NEXT:    vpsrlw %xmm1, %ymm0, %ymm5
; CHECK-AVX512-NEXT:    vpand %ymm2, %ymm3, %ymm2
; CHECK-AVX512-NEXT:    vpsrlw %xmm1, %xmm4, %xmm0
; CHECK-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm0
; CHECK-AVX512-NEXT:    vpbroadcastb %xmm0, %ymm0
; CHECK-AVX512-NEXT:    vpternlogq $236, %ymm5, %ymm2, %ymm0
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: splatvar_rotate_v32i8:
; CHECK-VBMI:       # %bb.0:
; CHECK-VBMI-NEXT:    vpbroadcastb %xmm1, %xmm1
; CHECK-VBMI-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
; CHECK-VBMI-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
; CHECK-VBMI-NEXT:    vpsllw %xmm2, %ymm0, %ymm3
; CHECK-VBMI-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
; CHECK-VBMI-NEXT:    vpsllw %xmm2, %xmm4, %xmm2
; CHECK-VBMI-NEXT:    vpbroadcastb %xmm2, %ymm2
; CHECK-VBMI-NEXT:    vpand %ymm2, %ymm3, %ymm2
; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
; CHECK-VBMI-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
; CHECK-VBMI-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
; CHECK-VBMI-NEXT:    vpsrlw %xmm1, %ymm0, %ymm3
; CHECK-VBMI-NEXT:    vpsrlw %xmm1, %xmm4, %xmm0
; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; CHECK-VBMI-NEXT:    vpermb %ymm0, %ymm1, %ymm0
; CHECK-VBMI-NEXT:    vpternlogq $236, %ymm3, %ymm2, %ymm0
; CHECK-VBMI-NEXT:    retq
  %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
  %splat8 = sub <32 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %splat
  %shl = shl <32 x i8> %a, %splat
  %lshr = lshr <32 x i8> %a, %splat8
  %or = or <32 x i8> %shl, %lshr
  ret <32 x i8> %or
}

define <32 x i8> @constant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
; CHECK-AVX512-LABEL: constant_rotate_v32i8:
; CHECK-AVX512:       # %bb.0:
; CHECK-AVX512-NEXT:    vpsllw $4, %ymm0, %ymm1
; CHECK-AVX512-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
; CHECK-AVX512-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
; CHECK-AVX512-NEXT:    # ymm2 = mem[0,1,0,1]
; CHECK-AVX512-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm1
; CHECK-AVX512-NEXT:    vpsllw $2, %ymm1, %ymm3
; CHECK-AVX512-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
; CHECK-AVX512-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
; CHECK-AVX512-NEXT:    vpblendvb %ymm2, %ymm3, %ymm1, %ymm1
; CHECK-AVX512-NEXT:    vpaddb %ymm1, %ymm1, %ymm3
; CHECK-AVX512-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
; CHECK-AVX512-NEXT:    vpblendvb %ymm2, %ymm3, %ymm1, %ymm1
; CHECK-AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
; CHECK-AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
; CHECK-AVX512-NEXT:    vpsllvw {{.*}}(%rip), %ymm3, %ymm3
; CHECK-AVX512-NEXT:    vpsrlw $8, %ymm3, %ymm3
; CHECK-AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
; CHECK-AVX512-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm0
; CHECK-AVX512-NEXT:    vpsrlw $8, %ymm0, %ymm0
; CHECK-AVX512-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
; CHECK-AVX512-NEXT:    vpor %ymm0, %ymm1, %ymm0
; CHECK-AVX512-NEXT:    retq
;
; CHECK-VBMI-LABEL: constant_rotate_v32i8:
; CHECK-VBMI:       # %bb.0:
; CHECK-VBMI-NEXT:    vpsllw $4, %ymm0, %ymm1
; CHECK-VBMI-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
; CHECK-VBMI-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
; CHECK-VBMI-NEXT:    # ymm2 = mem[0,1,0,1]
; CHECK-VBMI-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm1
; CHECK-VBMI-NEXT:    vpsllw $2, %ymm1, %ymm3
; CHECK-VBMI-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
; CHECK-VBMI-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
; CHECK-VBMI-NEXT:    vpblendvb %ymm2, %ymm3, %ymm1, %ymm1
; CHECK-VBMI-NEXT:    vpaddb %ymm1, %ymm1, %ymm3
; CHECK-VBMI-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
; CHECK-VBMI-NEXT:    vpblendvb %ymm2, %ymm3, %ymm1, %ymm1
; CHECK-VBMI-NEXT:    vpxor %xmm2, %xmm2, %xmm2
; CHECK-VBMI-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
; CHECK-VBMI-NEXT:    vpsllvw {{.*}}(%rip), %ymm3, %ymm3
; CHECK-VBMI-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
; CHECK-VBMI-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm0
; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,3,5,7,9,11,13,15,33,35,37,39,41,43,45,47,17,19,21,23,25,27,29,31,49,51,53,55,57,59,61,63]
; CHECK-VBMI-NEXT:    vpermi2b %ymm3, %ymm0, %ymm2
; CHECK-VBMI-NEXT:    vpor %ymm2, %ymm1, %ymm0
; CHECK-VBMI-NEXT:    retq
  %shl = shl <32 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>
  %lshr = lshr <32 x i8> %a, <i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
  %or = or <32 x i8> %shl, %lshr
  ret <32 x i8> %or
}

define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: splatconstant_rotate_v32i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpsllw $4, %ymm0, %ymm1
; CHECK-NEXT:    vpsrlw $4, %ymm0, %ymm0
; CHECK-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
; CHECK-NEXT:    retq
  %shl = shl <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
  %lshr = lshr <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
  %or = or <32 x i8> %shl, %lshr
  ret <32 x i8> %or
}

define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: splatconstant_rotate_mask_v32i8:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vpsllw $4, %ymm0, %ymm1
; CHECK-NEXT:    vpsrlw $4, %ymm0, %ymm0
; CHECK-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
; CHECK-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
; CHECK-NEXT:    retq
  %shl = shl <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
  %lshr = lshr <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
  %rmask = and <32 x i8> %lshr, <i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55>
  %lmask = and <32 x i8> %shl, <i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33>
  %or = or <32 x i8> %lmask, %rmask
  ret <32 x i8> %or
}