test/CodeGen/X86/combine-storetomstore.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx     | FileCheck %s -check-prefix=AVX
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2    | FileCheck %s -check-prefix=AVX2
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s -check-prefix=AVX512

define void @test_masked_store_success_v4i8(<4 x i8> %x, ptr %ptr, <4 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v4i8:
; AVX:       # %bb.0:
; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX-NEXT:    vpsllw $7, %xmm1, %xmm1
; AVX-NEXT:    vmovdqa (%rdi), %xmm2
; AVX-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
; AVX-NEXT:    vmovd %xmm0, (%rdi)
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v4i8:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
; AVX2-NEXT:    vpsllw $7, %xmm1, %xmm1
; AVX2-NEXT:    vmovdqa (%rdi), %xmm2
; AVX2-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
; AVX2-NEXT:    vmovd %xmm0, (%rdi)
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v4i8:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX512-NEXT:    vpmovd2m %xmm1, %k1
; AVX512-NEXT:    vmovdqa (%rdi), %xmm1
; AVX512-NEXT:    vmovdqu8 %xmm0, %xmm1 {%k1}
; AVX512-NEXT:    vmovd %xmm1, (%rdi)
; AVX512-NEXT:    retq
  %load = load <4 x i8>, ptr %ptr, align 32
  %sel = select <4 x i1> %mask, <4 x i8> %x, <4 x i8> %load
  store <4 x i8> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v4i16(<4 x i16> %x, ptr %ptr, <4 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v4i16:
; AVX:       # %bb.0:
; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
; AVX-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX-NEXT:    vpsraw $15, %xmm1, %xmm1
; AVX-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
; AVX-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
; AVX-NEXT:    vmovq %xmm0, (%rdi)
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v4i16:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
; AVX2-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX2-NEXT:    vpsraw $15, %xmm1, %xmm1
; AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
; AVX2-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
; AVX2-NEXT:    vmovq %xmm0, (%rdi)
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v4i16:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX512-NEXT:    vpmovd2m %xmm1, %k1
; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
; AVX512-NEXT:    vmovdqu16 %xmm0, %xmm1 {%k1}
; AVX512-NEXT:    vmovq %xmm1, (%rdi)
; AVX512-NEXT:    retq
  %load = load <4 x i16>, ptr %ptr, align 32
  %sel = select <4 x i1> %mask, <4 x i16> %x, <4 x i16> %load
  store <4 x i16> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v4i32(<4 x i32> %x, ptr %ptr, <4 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v4i32:
; AVX:       # %bb.0:
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vmaskmovps %xmm0, %xmm1, (%rdi)
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v4i32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX2-NEXT:    vpmaskmovd %xmm0, %xmm1, (%rdi)
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v4i32:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX512-NEXT:    vpmovd2m %xmm1, %k1
; AVX512-NEXT:    vmovdqa32 %xmm0, (%rdi) {%k1}
; AVX512-NEXT:    retq
  %load = load <4 x i32>, ptr %ptr, align 32
  %sel = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %load
  store <4 x i32> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v4i64(<4 x i64> %x, ptr %ptr, <4 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v4i64:
; AVX:       # %bb.0:
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vpmovsxdq %xmm1, %xmm2
; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm1, %xmm1
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vmaskmovpd %ymm0, %ymm1, (%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v4i64:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX2-NEXT:    vpmovsxdq %xmm1, %ymm1
; AVX2-NEXT:    vpmaskmovq %ymm0, %ymm1, (%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v4i64:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX512-NEXT:    vpmovd2m %xmm1, %k1
; AVX512-NEXT:    vmovdqa64 %ymm0, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <4 x i64>, ptr %ptr, align 32
  %sel = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> %load
  store <4 x i64> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v4f16(<4 x half> %x, ptr %ptr, <4 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v4f16:
; AVX:       # %bb.0:
; AVX-NEXT:    vpsrlq $48, %xmm0, %xmm2
; AVX-NEXT:    vpextrw $0, %xmm2, %edx
; AVX-NEXT:    vpsrld $16, %xmm0, %xmm2
; AVX-NEXT:    vpextrw $0, %xmm2, %ecx
; AVX-NEXT:    movzwl 2(%rdi), %eax
; AVX-NEXT:    vpextrb $4, %xmm1, %esi
; AVX-NEXT:    testb $1, %sil
; AVX-NEXT:    cmovnel %ecx, %eax
; AVX-NEXT:    vpextrb $8, %xmm1, %ecx
; AVX-NEXT:    testb $1, %cl
; AVX-NEXT:    jne .LBB4_1
; AVX-NEXT:  # %bb.2:
; AVX-NEXT:    movl 4(%rdi), %ecx
; AVX-NEXT:    jmp .LBB4_3
; AVX-NEXT:  .LBB4_1:
; AVX-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
; AVX-NEXT:    vpextrw $0, %xmm2, %ecx
; AVX-NEXT:  .LBB4_3:
; AVX-NEXT:    movzwl 6(%rdi), %esi
; AVX-NEXT:    vpextrb $12, %xmm1, %r8d
; AVX-NEXT:    testb $1, %r8b
; AVX-NEXT:    cmovnel %edx, %esi
; AVX-NEXT:    vmovd %xmm1, %edx
; AVX-NEXT:    testb $1, %dl
; AVX-NEXT:    jne .LBB4_4
; AVX-NEXT:  # %bb.5:
; AVX-NEXT:    movl (%rdi), %edx
; AVX-NEXT:    jmp .LBB4_6
; AVX-NEXT:  .LBB4_4:
; AVX-NEXT:    vpextrw $0, %xmm0, %edx
; AVX-NEXT:  .LBB4_6:
; AVX-NEXT:    movw %dx, (%rdi)
; AVX-NEXT:    movw %si, 6(%rdi)
; AVX-NEXT:    movw %cx, 4(%rdi)
; AVX-NEXT:    movw %ax, 2(%rdi)
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v4f16:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpsrlq $48, %xmm0, %xmm2
; AVX2-NEXT:    vpextrw $0, %xmm2, %edx
; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm2
; AVX2-NEXT:    vpextrw $0, %xmm2, %ecx
; AVX2-NEXT:    movzwl 2(%rdi), %eax
; AVX2-NEXT:    vpextrb $4, %xmm1, %esi
; AVX2-NEXT:    testb $1, %sil
; AVX2-NEXT:    cmovnel %ecx, %eax
; AVX2-NEXT:    vpextrb $8, %xmm1, %ecx
; AVX2-NEXT:    testb $1, %cl
; AVX2-NEXT:    jne .LBB4_1
; AVX2-NEXT:  # %bb.2:
; AVX2-NEXT:    movl 4(%rdi), %ecx
; AVX2-NEXT:    jmp .LBB4_3
; AVX2-NEXT:  .LBB4_1:
; AVX2-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
; AVX2-NEXT:    vpextrw $0, %xmm2, %ecx
; AVX2-NEXT:  .LBB4_3:
; AVX2-NEXT:    movzwl 6(%rdi), %esi
; AVX2-NEXT:    vpextrb $12, %xmm1, %r8d
; AVX2-NEXT:    testb $1, %r8b
; AVX2-NEXT:    cmovnel %edx, %esi
; AVX2-NEXT:    vmovd %xmm1, %edx
; AVX2-NEXT:    testb $1, %dl
; AVX2-NEXT:    jne .LBB4_4
; AVX2-NEXT:  # %bb.5:
; AVX2-NEXT:    movl (%rdi), %edx
; AVX2-NEXT:    jmp .LBB4_6
; AVX2-NEXT:  .LBB4_4:
; AVX2-NEXT:    vpextrw $0, %xmm0, %edx
; AVX2-NEXT:  .LBB4_6:
; AVX2-NEXT:    movw %dx, (%rdi)
; AVX2-NEXT:    movw %si, 6(%rdi)
; AVX2-NEXT:    movw %cx, 4(%rdi)
; AVX2-NEXT:    movw %ax, 2(%rdi)
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v4f16:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX512-NEXT:    vpmovd2m %xmm1, %k1
; AVX512-NEXT:    vmovdqa (%rdi), %xmm1
; AVX512-NEXT:    vmovdqu16 %xmm0, %xmm1 {%k1}
; AVX512-NEXT:    vmovq %xmm1, (%rdi)
; AVX512-NEXT:    retq
  %load = load <4 x half>, ptr %ptr, align 32
  %sel = select <4 x i1> %mask, <4 x half> %x, <4 x half> %load
  store <4 x half> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v4f32(<4 x float> %x, ptr %ptr, <4 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v4f32:
; AVX:       # %bb.0:
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vmaskmovps %xmm0, %xmm1, (%rdi)
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v4f32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX2-NEXT:    vmaskmovps %xmm0, %xmm1, (%rdi)
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v4f32:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX512-NEXT:    vpmovd2m %xmm1, %k1
; AVX512-NEXT:    vmovaps %xmm0, (%rdi) {%k1}
; AVX512-NEXT:    retq
  %load = load <4 x float>, ptr %ptr, align 32
  %sel = select <4 x i1> %mask, <4 x float> %x, <4 x float> %load
  store <4 x float> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v4f64(<4 x double> %x, ptr %ptr, <4 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v4f64:
; AVX:       # %bb.0:
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vpmovsxdq %xmm1, %xmm2
; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm1, %xmm1
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vmaskmovpd %ymm0, %ymm1, (%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v4f64:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX2-NEXT:    vpmovsxdq %xmm1, %ymm1
; AVX2-NEXT:    vmaskmovpd %ymm0, %ymm1, (%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v4f64:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX512-NEXT:    vpmovd2m %xmm1, %k1
; AVX512-NEXT:    vmovapd %ymm0, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <4 x double>, ptr %ptr, align 32
  %sel = select <4 x i1> %mask, <4 x double> %x, <4 x double> %load
  store <4 x double> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v8i8(<8 x i8> %x, ptr %ptr, <8 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v8i8:
; AVX:       # %bb.0:
; AVX-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
; AVX-NEXT:    vpsllw $7, %xmm1, %xmm1
; AVX-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
; AVX-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
; AVX-NEXT:    vmovq %xmm0, (%rdi)
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v8i8:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
; AVX2-NEXT:    vpsllw $7, %xmm1, %xmm1
; AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
; AVX2-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
; AVX2-NEXT:    vmovq %xmm0, (%rdi)
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v8i8:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX512-NEXT:    vpmovw2m %xmm1, %k1
; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
; AVX512-NEXT:    vmovdqu8 %xmm0, %xmm1 {%k1}
; AVX512-NEXT:    vmovq %xmm1, (%rdi)
; AVX512-NEXT:    retq
  %load = load <8 x i8>, ptr %ptr, align 32
  %sel = select <8 x i1> %mask, <8 x i8> %x, <8 x i8> %load
  store <8 x i8> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v8i16(<8 x i16> %x, ptr %ptr, <8 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v8i16:
; AVX:       # %bb.0:
; AVX-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX-NEXT:    vpsraw $15, %xmm1, %xmm1
; AVX-NEXT:    vmovdqa (%rdi), %xmm2
; AVX-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
; AVX-NEXT:    vmovdqa %xmm0, (%rdi)
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v8i16:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX2-NEXT:    vpsraw $15, %xmm1, %xmm1
; AVX2-NEXT:    vmovdqa (%rdi), %xmm2
; AVX2-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v8i16:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX512-NEXT:    vpmovw2m %xmm1, %k1
; AVX512-NEXT:    vmovdqu16 %xmm0, (%rdi) {%k1}
; AVX512-NEXT:    retq
  %load = load <8 x i16>, ptr %ptr, align 32
  %sel = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> %load
  store <8 x i16> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v8i32:
; AVX:       # %bb.0:
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vmaskmovps %ymm0, %ymm1, (%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v8i32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
; AVX2-NEXT:    vpmaskmovd %ymm0, %ymm1, (%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v8i32:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX512-NEXT:    vpmovw2m %xmm1, %k1
; AVX512-NEXT:    vmovdqa32 %ymm0, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <8 x i32>, ptr %ptr, align 32
  %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load
  store <8 x i32> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v8i64(<8 x i64> %x, ptr %ptr, <8 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v8i64:
; AVX:       # %bb.0:
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm3, %xmm3
; AVX-NEXT:    vpmovsxdq %xmm3, %xmm4
; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm3, %xmm3
; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX-NEXT:    vpmovsxdq %xmm2, %xmm4
; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm2, %xmm2
; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
; AVX-NEXT:    vmaskmovpd %ymm0, %ymm2, (%rdi)
; AVX-NEXT:    vmaskmovpd %ymm1, %ymm3, 32(%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v8i64:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]
; AVX2-NEXT:    vpslld $31, %xmm3, %xmm3
; AVX2-NEXT:    vpmovsxdq %xmm3, %ymm3
; AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; AVX2-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX2-NEXT:    vpmovsxdq %xmm2, %ymm2
; AVX2-NEXT:    vpmaskmovq %ymm0, %ymm2, (%rdi)
; AVX2-NEXT:    vpmaskmovq %ymm1, %ymm3, 32(%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v8i64:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX512-NEXT:    vpmovw2m %xmm1, %k1
; AVX512-NEXT:    vmovdqu64 %zmm0, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <8 x i64>, ptr %ptr, align 32
  %sel = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> %load
  store <8 x i64> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v8f16(<8 x half> %x, ptr %ptr, <8 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v8f16:
; AVX:       # %bb.0:
; AVX-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX-NEXT:    vpsraw $15, %xmm1, %xmm1
; AVX-NEXT:    vmovdqa (%rdi), %xmm2
; AVX-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
; AVX-NEXT:    vmovdqa %xmm0, (%rdi)
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v8f16:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX2-NEXT:    vpsraw $15, %xmm1, %xmm1
; AVX2-NEXT:    vmovdqa (%rdi), %xmm2
; AVX2-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v8f16:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX512-NEXT:    vpmovw2m %xmm1, %k1
; AVX512-NEXT:    vmovdqu16 %xmm0, (%rdi) {%k1}
; AVX512-NEXT:    retq
  %load = load <8 x half>, ptr %ptr, align 32
  %sel = select <8 x i1> %mask, <8 x half> %x, <8 x half> %load
  store <8 x half> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v8f32(<8 x float> %x, ptr %ptr, <8 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v8f32:
; AVX:       # %bb.0:
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vmaskmovps %ymm0, %ymm1, (%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v8f32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
; AVX2-NEXT:    vmaskmovps %ymm0, %ymm1, (%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v8f32:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX512-NEXT:    vpmovw2m %xmm1, %k1
; AVX512-NEXT:    vmovaps %ymm0, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <8 x float>, ptr %ptr, align 32
  %sel = select <8 x i1> %mask, <8 x float> %x, <8 x float> %load
  store <8 x float> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v8f64(<8 x double> %x, ptr %ptr, <8 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v8f64:
; AVX:       # %bb.0:
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm3, %xmm3
; AVX-NEXT:    vpmovsxdq %xmm3, %xmm4
; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm3, %xmm3
; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX-NEXT:    vpmovsxdq %xmm2, %xmm4
; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm2, %xmm2
; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
; AVX-NEXT:    vmaskmovpd %ymm0, %ymm2, (%rdi)
; AVX-NEXT:    vmaskmovpd %ymm1, %ymm3, 32(%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v8f64:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]
; AVX2-NEXT:    vpslld $31, %xmm3, %xmm3
; AVX2-NEXT:    vpmovsxdq %xmm3, %ymm3
; AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; AVX2-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX2-NEXT:    vpmovsxdq %xmm2, %ymm2
; AVX2-NEXT:    vmaskmovpd %ymm0, %ymm2, (%rdi)
; AVX2-NEXT:    vmaskmovpd %ymm1, %ymm3, 32(%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v8f64:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX512-NEXT:    vpmovw2m %xmm1, %k1
; AVX512-NEXT:    vmovupd %zmm0, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <8 x double>, ptr %ptr, align 32
  %sel = select <8 x i1> %mask, <8 x double> %x, <8 x double> %load
  store <8 x double> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v16i8(<16 x i8> %x, ptr %ptr, <16 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v16i8:
; AVX:       # %bb.0:
; AVX-NEXT:    vpsllw $7, %xmm1, %xmm1
; AVX-NEXT:    vmovdqa (%rdi), %xmm2
; AVX-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
; AVX-NEXT:    vmovdqa %xmm0, (%rdi)
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v16i8:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpsllw $7, %xmm1, %xmm1
; AVX2-NEXT:    vmovdqa (%rdi), %xmm2
; AVX2-NEXT:    vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v16i8:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $7, %xmm1, %xmm1
; AVX512-NEXT:    vpmovb2m %xmm1, %k1
; AVX512-NEXT:    vmovdqu8 %xmm0, (%rdi) {%k1}
; AVX512-NEXT:    retq
  %load = load <16 x i8>, ptr %ptr, align 32
  %sel = select <16 x i1> %mask, <16 x i8> %x, <16 x i8> %load
  store <16 x i8> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v16i16(<16 x i16> %x, ptr %ptr, <16 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v16i16:
; AVX:       # %bb.0:
; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; AVX-NEXT:    vpsllw $15, %xmm2, %xmm2
; AVX-NEXT:    vpsraw $15, %xmm2, %xmm2
; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX-NEXT:    vpsraw $15, %xmm1, %xmm1
; AVX-NEXT:    vmovdqa (%rdi), %xmm3
; AVX-NEXT:    vpblendvb %xmm1, %xmm0, %xmm3, %xmm1
; AVX-NEXT:    vmovdqa 16(%rdi), %xmm3
; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
; AVX-NEXT:    vpblendvb %xmm2, %xmm0, %xmm3, %xmm0
; AVX-NEXT:    vmovdqa %xmm1, (%rdi)
; AVX-NEXT:    vmovdqa %xmm0, 16(%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v16i16:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
; AVX2-NEXT:    vpsllw $15, %ymm1, %ymm1
; AVX2-NEXT:    vpsraw $15, %ymm1, %ymm1
; AVX2-NEXT:    vmovdqa (%rdi), %ymm2
; AVX2-NEXT:    vpblendvb %ymm1, %ymm0, %ymm2, %ymm0
; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v16i16:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $7, %xmm1, %xmm1
; AVX512-NEXT:    vpmovb2m %xmm1, %k1
; AVX512-NEXT:    vmovdqu16 %ymm0, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <16 x i16>, ptr %ptr, align 32
  %sel = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> %load
  store <16 x i16> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v16i32(<16 x i32> %x, ptr %ptr, <16 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v16i32:
; AVX:       # %bb.0:
; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
; AVX-NEXT:    vpslld $31, %xmm4, %xmm4
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm3, %xmm3
; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
; AVX-NEXT:    vpslld $31, %xmm4, %xmm4
; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
; AVX-NEXT:    vmaskmovps %ymm0, %ymm2, (%rdi)
; AVX-NEXT:    vmaskmovps %ymm1, %ymm3, 32(%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v16i32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
; AVX2-NEXT:    vpslld $31, %ymm3, %ymm3
; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
; AVX2-NEXT:    vpslld $31, %ymm2, %ymm2
; AVX2-NEXT:    vpmaskmovd %ymm0, %ymm2, (%rdi)
; AVX2-NEXT:    vpmaskmovd %ymm1, %ymm3, 32(%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v16i32:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $7, %xmm1, %xmm1
; AVX512-NEXT:    vpmovb2m %xmm1, %k1
; AVX512-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <16 x i32>, ptr %ptr, align 32
  %sel = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %load
  store <16 x i32> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v32i8(<32 x i8> %x, ptr %ptr, <32 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v32i8:
; AVX:       # %bb.0:
; AVX-NEXT:    vpsllw $7, %xmm1, %xmm2
; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
; AVX-NEXT:    vpsllw $7, %xmm1, %xmm1
; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm3
; AVX-NEXT:    vmovdqa (%rdi), %xmm4
; AVX-NEXT:    vmovdqa 16(%rdi), %xmm5
; AVX-NEXT:    vpblendvb %xmm1, %xmm3, %xmm5, %xmm1
; AVX-NEXT:    vpblendvb %xmm2, %xmm0, %xmm4, %xmm0
; AVX-NEXT:    vmovdqa %xmm0, (%rdi)
; AVX-NEXT:    vmovdqa %xmm1, 16(%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v32i8:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpsllw $7, %ymm1, %ymm1
; AVX2-NEXT:    vmovdqa (%rdi), %ymm2
; AVX2-NEXT:    vpblendvb %ymm1, %ymm0, %ymm2, %ymm0
; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v32i8:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $7, %ymm1, %ymm1
; AVX512-NEXT:    vpmovb2m %ymm1, %k1
; AVX512-NEXT:    vmovdqu8 %ymm0, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <32 x i8>, ptr %ptr, align 32
  %sel = select <32 x i1> %mask, <32 x i8> %x, <32 x i8> %load
  store <32 x i8> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v32i16(<32 x i16> %x, ptr %ptr, <32 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v32i16:
; AVX:       # %bb.0:
; AVX-NEXT:    vmovdqa (%rdi), %xmm3
; AVX-NEXT:    vmovdqa 16(%rdi), %xmm4
; AVX-NEXT:    vmovdqa 32(%rdi), %xmm5
; AVX-NEXT:    vmovdqa 48(%rdi), %xmm6
; AVX-NEXT:    vextractf128 $1, %ymm2, %xmm7
; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
; AVX-NEXT:    vpsllw $15, %xmm8, %xmm8
; AVX-NEXT:    vpsraw $15, %xmm8, %xmm8
; AVX-NEXT:    vpblendvb %xmm8, %xmm1, %xmm5, %xmm5
; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm8 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
; AVX-NEXT:    vpsllw $15, %xmm8, %xmm8
; AVX-NEXT:    vpsraw $15, %xmm8, %xmm8
; AVX-NEXT:    vpblendvb %xmm8, %xmm0, %xmm3, %xmm3
; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; AVX-NEXT:    vpsllw $15, %xmm7, %xmm7
; AVX-NEXT:    vpsraw $15, %xmm7, %xmm7
; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
; AVX-NEXT:    vpblendvb %xmm7, %xmm1, %xmm6, %xmm1
; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; AVX-NEXT:    vpsllw $15, %xmm2, %xmm2
; AVX-NEXT:    vpsraw $15, %xmm2, %xmm2
; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
; AVX-NEXT:    vpblendvb %xmm2, %xmm0, %xmm4, %xmm0
; AVX-NEXT:    vmovdqa %xmm3, (%rdi)
; AVX-NEXT:    vmovdqa %xmm0, 16(%rdi)
; AVX-NEXT:    vmovdqa %xmm5, 32(%rdi)
; AVX-NEXT:    vmovdqa %xmm1, 48(%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v32i16:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vmovdqa (%rdi), %ymm3
; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm4
; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm5
; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
; AVX2-NEXT:    vpsllw $15, %ymm5, %ymm5
; AVX2-NEXT:    vpsraw $15, %ymm5, %ymm5
; AVX2-NEXT:    vpblendvb %ymm5, %ymm1, %ymm4, %ymm1
; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
; AVX2-NEXT:    vpsllw $15, %ymm2, %ymm2
; AVX2-NEXT:    vpsraw $15, %ymm2, %ymm2
; AVX2-NEXT:    vpblendvb %ymm2, %ymm0, %ymm3, %ymm0
; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
; AVX2-NEXT:    vmovdqa %ymm1, 32(%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v32i16:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $7, %ymm1, %ymm1
; AVX512-NEXT:    vpmovb2m %ymm1, %k1
; AVX512-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <32 x i16>, ptr %ptr, align 32
  %sel = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> %load
  store <32 x i16> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_v64i8(<64 x i8> %x, ptr %ptr, <64 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v64i8:
; AVX:       # %bb.0:
; AVX-NEXT:    vmovd %esi, %xmm2
; AVX-NEXT:    vpinsrb $1, %edx, %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $2, %ecx, %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $3, %r8d, %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $4, %r9d, %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX-NEXT:    vpsllw $7, %xmm2, %xmm2
; AVX-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
; AVX-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX-NEXT:    vpsllw $7, %xmm3, %xmm3
; AVX-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
; AVX-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX-NEXT:    vpsllw $7, %xmm4, %xmm4
; AVX-NEXT:    vmovd {{.*#+}} xmm5 = mem[0],zero,zero,zero
; AVX-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm5, %xmm5
; AVX-NEXT:    vpsllw $7, %xmm5, %xmm5
; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm6
; AVX-NEXT:    vmovdqa (%rdi), %xmm7
; AVX-NEXT:    vmovdqa 16(%rdi), %xmm8
; AVX-NEXT:    vmovdqa 32(%rdi), %xmm9
; AVX-NEXT:    vmovdqa 48(%rdi), %xmm10
; AVX-NEXT:    vpblendvb %xmm5, %xmm6, %xmm10, %xmm5
; AVX-NEXT:    vpblendvb %xmm4, %xmm1, %xmm9, %xmm1
; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm4
; AVX-NEXT:    vpblendvb %xmm3, %xmm4, %xmm8, %xmm3
; AVX-NEXT:    vpblendvb %xmm2, %xmm0, %xmm7, %xmm0
; AVX-NEXT:    vmovdqa %xmm3, 16(%rdi)
; AVX-NEXT:    vmovdqa %xmm1, 32(%rdi)
; AVX-NEXT:    vmovdqa %xmm5, 48(%rdi)
; AVX-NEXT:    vmovdqa %xmm0, (%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_v64i8:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
; AVX2-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2
; AVX2-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
; AVX2-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
; AVX2-NEXT:    vpsllw $7, %ymm2, %ymm2
; AVX2-NEXT:    vmovd %esi, %xmm3
; AVX2-NEXT:    vpinsrb $1, %edx, %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $2, %ecx, %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $3, %r8d, %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $4, %r9d, %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm3, %xmm3
; AVX2-NEXT:    vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
; AVX2-NEXT:    vpinsrb $1, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $2, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $3, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $4, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $5, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $6, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $7, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $8, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $9, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $10, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $11, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $12, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $13, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $14, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vpinsrb $15, {{[0-9]+}}(%rsp), %xmm4, %xmm4
; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
; AVX2-NEXT:    vpsllw $7, %ymm3, %ymm3
; AVX2-NEXT:    vmovdqa (%rdi), %ymm4
; AVX2-NEXT:    vpblendvb %ymm3, %ymm0, %ymm4, %ymm0
; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm3
; AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm3, %ymm1
; AVX2-NEXT:    vmovdqa %ymm1, 32(%rdi)
; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_v64i8:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $7, %zmm1, %zmm1
; AVX512-NEXT:    vpmovb2m %zmm1, %k1
; AVX512-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <64 x i8>, ptr %ptr, align 32
  %sel = select <64 x i1> %mask, <64 x i8> %x, <64 x i8> %load
  store <64 x i8> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_invert_mask_v4i32(<4 x i32> %x, ptr %ptr, <4 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_invert_mask_v4i32:
; AVX:       # %bb.0:
; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vmaskmovps %xmm0, %xmm1, (%rdi)
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_invert_mask_v4i32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX2-NEXT:    vpmaskmovd %xmm0, %xmm1, (%rdi)
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_invert_mask_v4i32:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX512-NEXT:    vpmovd2m %xmm1, %k0
; AVX512-NEXT:    knotw %k0, %k1
; AVX512-NEXT:    vmovdqa32 %xmm0, (%rdi) {%k1}
; AVX512-NEXT:    retq
  %load = load <4 x i32>, ptr %ptr, align 32
  %sel = select <4 x i1> %mask, <4 x i32> %load, <4 x i32> %x
  store <4 x i32> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_invert_mask_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_invert_mask_v8i32:
; AVX:       # %bb.0:
; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
; AVX-NEXT:    vpxor %xmm2, %xmm1, %xmm1
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vmaskmovps %ymm0, %ymm1, (%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_invert_mask_v8i32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
; AVX2-NEXT:    vpmaskmovd %ymm0, %ymm1, (%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_invert_mask_v8i32:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX512-NEXT:    vpmovw2m %xmm1, %k0
; AVX512-NEXT:    knotb %k0, %k1
; AVX512-NEXT:    vmovdqa32 %ymm0, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <8 x i32>, ptr %ptr, align 32
  %sel = select <8 x i1> %mask, <8 x i32> %load, <8 x i32> %x
  store <8 x i32> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_success_invert_mask_v16i32(<16 x i32> %x, ptr %ptr, <16 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_invert_mask_v16i32:
; AVX:       # %bb.0:
; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; AVX-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
; AVX-NEXT:    vpxor %xmm5, %xmm2, %xmm2
; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX-NEXT:    vpxor %xmm5, %xmm4, %xmm4
; AVX-NEXT:    vpslld $31, %xmm4, %xmm4
; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
; AVX-NEXT:    vmaskmovps %ymm1, %ymm2, 32(%rdi)
; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
; AVX-NEXT:    vcmptrueps %ymm1, %ymm1, %ymm1
; AVX-NEXT:    vxorps %ymm1, %ymm3, %ymm1
; AVX-NEXT:    vpslld $31, %xmm1, %xmm2
; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm1
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vmaskmovps %ymm0, %ymm1, (%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_success_invert_mask_v16i32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
; AVX2-NEXT:    vpxor %ymm4, %ymm2, %ymm2
; AVX2-NEXT:    vpslld $31, %ymm2, %ymm2
; AVX2-NEXT:    vpmaskmovd %ymm0, %ymm2, (%rdi)
; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm0
; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
; AVX2-NEXT:    vpmaskmovd %ymm1, %ymm0, 32(%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_success_invert_mask_v16i32:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $7, %xmm1, %xmm1
; AVX512-NEXT:    vpmovb2m %xmm1, %k0
; AVX512-NEXT:    knotw %k0, %k1
; AVX512-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <16 x i32>, ptr %ptr, align 32
  %sel = select <16 x i1> %mask, <16 x i32> %load, <16 x i32> %x
  store <16 x i32> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_zextload(<4 x i64> %x, ptr %ptr, <4 x i1> %mask) {
; AVX-LABEL: test_masked_store_zextload:
; AVX:       # %bb.0:
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vpmovsxdq %xmm1, %xmm2
; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm1, %xmm1
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm2 = mem[0],zero,mem[1],zero
; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm3 = mem[0],zero,mem[1],zero
; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
; AVX-NEXT:    vblendvpd %ymm1, %ymm0, %ymm2, %ymm0
; AVX-NEXT:    vmovapd %ymm0, (%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_zextload:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX2-NEXT:    vpmovsxdq %xmm1, %ymm1
; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
; AVX2-NEXT:    vblendvpd %ymm1, %ymm0, %ymm2, %ymm0
; AVX2-NEXT:    vmovapd %ymm0, (%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_zextload:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX512-NEXT:    vpmovd2m %xmm1, %k1
; AVX512-NEXT:    vpmovzxdq {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm1 {%k1}
; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <4 x i32>, ptr %ptr, align 32
  %zext = zext <4 x i32> %load to <4 x i64>
  %masked = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> %zext
  store <4 x i64> %masked, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_volatile_load(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; AVX-LABEL: test_masked_store_volatile_load:
; AVX:       # %bb.0:
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vmovaps (%rdi), %ymm2
; AVX-NEXT:    vblendvps %ymm1, %ymm0, %ymm2, %ymm0
; AVX-NEXT:    vmovaps %ymm0, (%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_volatile_load:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
; AVX2-NEXT:    vmovaps (%rdi), %ymm2
; AVX2-NEXT:    vblendvps %ymm1, %ymm0, %ymm2, %ymm0
; AVX2-NEXT:    vmovaps %ymm0, (%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_volatile_load:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX512-NEXT:    vpmovw2m %xmm1, %k1
; AVX512-NEXT:    vmovdqa (%rdi), %ymm1
; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load volatile <8 x i32>, ptr %ptr, align 32
  %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load
  store <8 x i32> %sel, ptr %ptr, align 32
  ret void
}

define void @test_masked_store_volatile_store(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {
; AVX-LABEL: test_masked_store_volatile_store:
; AVX:       # %bb.0:
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vmovaps (%rdi), %ymm2
; AVX-NEXT:    vblendvps %ymm1, %ymm0, %ymm2, %ymm0
; AVX-NEXT:    vmovaps %ymm0, (%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_volatile_store:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
; AVX2-NEXT:    vmovaps (%rdi), %ymm2
; AVX2-NEXT:    vblendvps %ymm1, %ymm0, %ymm2, %ymm0
; AVX2-NEXT:    vmovaps %ymm0, (%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_volatile_store:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX512-NEXT:    vpmovw2m %xmm1, %k1
; AVX512-NEXT:    vmovdqa (%rdi), %ymm1
; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <8 x i32>, ptr %ptr, align 32
  %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load
  store volatile <8 x i32> %sel, ptr %ptr, align 32
  ret void
}

declare void @use_vec(<8 x i32>)

define void @test_masked_store_intervening(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) nounwind {
; AVX-LABEL: test_masked_store_intervening:
; AVX:       # %bb.0:
; AVX-NEXT:    pushq %rbx
; AVX-NEXT:    subq $32, %rsp
; AVX-NEXT:    movq %rdi, %rbx
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vmovaps (%rdi), %ymm2
; AVX-NEXT:    vblendvps %ymm1, %ymm0, %ymm2, %ymm0
; AVX-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
; AVX-NEXT:    vmovaps %ymm0, (%rdi)
; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
; AVX-NEXT:    callq use_vec@PLT
; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
; AVX-NEXT:    vmovaps %ymm0, (%rbx)
; AVX-NEXT:    addq $32, %rsp
; AVX-NEXT:    popq %rbx
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_intervening:
; AVX2:       # %bb.0:
; AVX2-NEXT:    pushq %rbx
; AVX2-NEXT:    subq $32, %rsp
; AVX2-NEXT:    movq %rdi, %rbx
; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
; AVX2-NEXT:    vmovaps (%rdi), %ymm2
; AVX2-NEXT:    vblendvps %ymm1, %ymm0, %ymm2, %ymm0
; AVX2-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
; AVX2-NEXT:    vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT:    vmovaps %ymm0, (%rdi)
; AVX2-NEXT:    vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT:    callq use_vec@PLT
; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
; AVX2-NEXT:    vmovaps %ymm0, (%rbx)
; AVX2-NEXT:    addq $32, %rsp
; AVX2-NEXT:    popq %rbx
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_intervening:
; AVX512:       # %bb.0:
; AVX512-NEXT:    pushq %rbx
; AVX512-NEXT:    subq $80, %rsp
; AVX512-NEXT:    movq %rdi, %rbx
; AVX512-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT:    vpsllw $15, %xmm1, %xmm0
; AVX512-NEXT:    vpmovw2m %xmm0, %k1
; AVX512-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; AVX512-NEXT:    vmovaps (%rdi), %ymm0
; AVX512-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT:    vxorps %xmm0, %xmm0, %xmm0
; AVX512-NEXT:    vmovaps %ymm0, (%rdi)
; AVX512-NEXT:    vxorps %xmm0, %xmm0, %xmm0
; AVX512-NEXT:    callq use_vec@PLT
; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
; AVX512-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
; AVX512-NEXT:    vmovdqa %ymm1, (%rbx)
; AVX512-NEXT:    addq $80, %rsp
; AVX512-NEXT:    popq %rbx
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <8 x i32>, ptr %ptr, align 32
  store <8 x i32> zeroinitializer, ptr %ptr, align 32
  %tmp = load <8 x i32>, ptr %ptr
  call void @use_vec(<8 x i32> %tmp)
  %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load
  store <8 x i32> %sel, ptr %ptr, align 32
  ret void
}


define void @test_masked_store_multiple_v8i32(<8 x i32> %x, <8 x i32> %y, ptr %ptr1, ptr %ptr2, <8 x i1> %mask, <8 x i1> %mask2) {
; AVX-LABEL: test_masked_store_multiple_v8i32:
; AVX:       # %bb.0:
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; AVX-NEXT:    vpslld $31, %xmm4, %xmm4
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
; AVX-NEXT:    vpslld $31, %xmm4, %xmm4
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm3, %xmm3
; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
; AVX-NEXT:    vmovaps (%rsi), %ymm4
; AVX-NEXT:    vblendvps %ymm3, %ymm1, %ymm4, %ymm1
; AVX-NEXT:    vmaskmovps %ymm0, %ymm2, (%rdi)
; AVX-NEXT:    vmovaps %ymm1, (%rsi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_multiple_v8i32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
; AVX2-NEXT:    vpslld $31, %ymm2, %ymm2
; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
; AVX2-NEXT:    vpslld $31, %ymm3, %ymm3
; AVX2-NEXT:    vmovaps (%rsi), %ymm4
; AVX2-NEXT:    vblendvps %ymm3, %ymm1, %ymm4, %ymm1
; AVX2-NEXT:    vpmaskmovd %ymm0, %ymm2, (%rdi)
; AVX2-NEXT:    vmovaps %ymm1, (%rsi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_multiple_v8i32:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm2, %xmm2
; AVX512-NEXT:    vpmovw2m %xmm2, %k1
; AVX512-NEXT:    vpsllw $15, %xmm3, %xmm2
; AVX512-NEXT:    vmovdqa (%rsi), %ymm3
; AVX512-NEXT:    vpmovw2m %xmm2, %k2
; AVX512-NEXT:    vmovdqa32 %ymm1, %ymm3 {%k2}
; AVX512-NEXT:    vmovdqa32 %ymm0, (%rdi) {%k1}
; AVX512-NEXT:    vmovdqa %ymm3, (%rsi)
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <8 x i32>, ptr %ptr1, align 32
  %load2 = load <8 x i32>, ptr %ptr2, align 32
  %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load
  %sel2 = select <8 x i1> %mask2, <8 x i32> %y, <8 x i32> %load2
  store <8 x i32> %sel, ptr %ptr1, align 32
  store <8 x i32> %sel2, ptr %ptr2, align 32
  ret void
}

define void @test_masked_store_multiple_v8i64(<8 x i64> %x, <8 x i64> %y, ptr %ptr1, ptr %ptr2, <8 x i1> %mask, <8 x i1> %mask2) {
; AVX-LABEL: test_masked_store_multiple_v8i64:
; AVX:       # %bb.0:
; AVX-NEXT:    vmovapd (%rsi), %ymm6
; AVX-NEXT:    vmovapd 32(%rsi), %ymm7
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm4[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm8, %xmm8
; AVX-NEXT:    vpmovsxdq %xmm8, %xmm9
; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm8, %xmm8
; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
; AVX-NEXT:    vpslld $31, %xmm4, %xmm4
; AVX-NEXT:    vpmovsxdq %xmm4, %xmm9
; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm4, %xmm4
; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm9, %ymm4
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm5[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm9, %xmm9
; AVX-NEXT:    vpmovsxdq %xmm9, %xmm10
; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm9, %xmm9
; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm9
; AVX-NEXT:    vblendvpd %ymm9, %ymm3, %ymm7, %ymm3
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
; AVX-NEXT:    vpslld $31, %xmm5, %xmm5
; AVX-NEXT:    vpmovsxdq %xmm5, %xmm7
; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm5, %xmm5
; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm7, %ymm5
; AVX-NEXT:    vblendvpd %ymm5, %ymm2, %ymm6, %ymm2
; AVX-NEXT:    vmaskmovpd %ymm0, %ymm4, (%rdi)
; AVX-NEXT:    vmaskmovpd %ymm1, %ymm8, 32(%rdi)
; AVX-NEXT:    vmovapd %ymm3, 32(%rsi)
; AVX-NEXT:    vmovapd %ymm2, (%rsi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_multiple_v8i64:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vmovapd (%rsi), %ymm6
; AVX2-NEXT:    vmovapd 32(%rsi), %ymm7
; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm4[4,4,5,5,6,6,7,7]
; AVX2-NEXT:    vpslld $31, %xmm8, %xmm8
; AVX2-NEXT:    vpmovsxdq %xmm8, %ymm8
; AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
; AVX2-NEXT:    vpslld $31, %xmm4, %xmm4
; AVX2-NEXT:    vpmovsxdq %xmm4, %ymm4
; AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm9 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
; AVX2-NEXT:    vpslld $31, %xmm9, %xmm9
; AVX2-NEXT:    vpmovsxdq %xmm9, %ymm9
; AVX2-NEXT:    vblendvpd %ymm9, %ymm2, %ymm6, %ymm2
; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
; AVX2-NEXT:    vpslld $31, %xmm5, %xmm5
; AVX2-NEXT:    vpmovsxdq %xmm5, %ymm5
; AVX2-NEXT:    vblendvpd %ymm5, %ymm3, %ymm7, %ymm3
; AVX2-NEXT:    vpmaskmovq %ymm0, %ymm4, (%rdi)
; AVX2-NEXT:    vpmaskmovq %ymm1, %ymm8, 32(%rdi)
; AVX2-NEXT:    vmovapd %ymm3, 32(%rsi)
; AVX2-NEXT:    vmovapd %ymm2, (%rsi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_multiple_v8i64:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm2, %xmm2
; AVX512-NEXT:    vpmovw2m %xmm2, %k1
; AVX512-NEXT:    vpsllw $15, %xmm3, %xmm2
; AVX512-NEXT:    vmovdqu64 (%rsi), %zmm3
; AVX512-NEXT:    vpmovw2m %xmm2, %k2
; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k2}
; AVX512-NEXT:    vmovdqu64 %zmm0, (%rdi) {%k1}
; AVX512-NEXT:    vmovdqu64 %zmm3, (%rsi)
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %load = load <8 x i64>, ptr %ptr1, align 32
  %load2 = load <8 x i64>, ptr %ptr2, align 32
  %sel = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> %load
  %sel2 = select <8 x i1> %mask2, <8 x i64> %y, <8 x i64> %load2
  store <8 x i64> %sel, ptr %ptr1, align 32
  store <8 x i64> %sel2, ptr %ptr2, align 32
  ret void
}

define void @test_masked_store_unaligned_v4i32(<4 x i32> %data, ptr %ptr, <4 x i1> %mask) {
; AVX-LABEL: test_masked_store_unaligned_v4i32:
; AVX:       # %bb.0:
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vmaskmovps %xmm0, %xmm1, 1(%rdi)
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_unaligned_v4i32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX2-NEXT:    vpmaskmovd %xmm0, %xmm1, 1(%rdi)
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_unaligned_v4i32:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX512-NEXT:    vpmovd2m %xmm1, %k1
; AVX512-NEXT:    vmovdqu32 %xmm0, 1(%rdi) {%k1}
; AVX512-NEXT:    retq
  %ptr_i8 = getelementptr i8, ptr %ptr, i32 1
  %ptr_vec = bitcast ptr %ptr_i8 to ptr
  %load = load <4 x i32>, ptr %ptr_vec, align 1
  %sel = select <4 x i1> %mask, <4 x i32> %data, <4 x i32> %load
  store <4 x i32> %sel, ptr %ptr_vec, align 1
  ret void
}

define void @test_masked_store_unaligned_v4i64(<4 x i64> %data, ptr %ptr, <4 x i1> %mask) {
; AVX-LABEL: test_masked_store_unaligned_v4i64:
; AVX:       # %bb.0:
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vpmovsxdq %xmm1, %xmm2
; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm1, %xmm1
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vmaskmovpd %ymm0, %ymm1, 1(%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_unaligned_v4i64:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX2-NEXT:    vpmovsxdq %xmm1, %ymm1
; AVX2-NEXT:    vpmaskmovq %ymm0, %ymm1, 1(%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_unaligned_v4i64:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX512-NEXT:    vpmovd2m %xmm1, %k1
; AVX512-NEXT:    vmovdqu64 %ymm0, 1(%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %ptr_i8 = getelementptr i8, ptr %ptr, i64 1
  %ptr_vec = bitcast ptr %ptr_i8 to ptr
  %load = load <4 x i64>, ptr %ptr_vec, align 1
  %sel = select <4 x i1> %mask, <4 x i64> %data, <4 x i64> %load
  store <4 x i64> %sel, ptr %ptr_vec, align 1
  ret void
}

define void @test_masked_store_unaligned_v8i32(<8 x i32> %data, ptr %ptr, <8 x i1> %mask) {
; AVX-LABEL: test_masked_store_unaligned_v8i32:
; AVX:       # %bb.0:
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
; AVX-NEXT:    vmaskmovps %ymm0, %ymm1, 1(%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_unaligned_v8i32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
; AVX2-NEXT:    vpmaskmovd %ymm0, %ymm1, 1(%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_unaligned_v8i32:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX512-NEXT:    vpmovw2m %xmm1, %k1
; AVX512-NEXT:    vmovdqu32 %ymm0, 1(%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %ptr_i8 = getelementptr i8, ptr %ptr, i32 1
  %ptr_vec = bitcast ptr %ptr_i8 to ptr
  %load = load <8 x i32>, ptr %ptr_vec, align 1
  %sel = select <8 x i1> %mask, <8 x i32> %data, <8 x i32> %load
  store <8 x i32> %sel, ptr %ptr_vec, align 1
  ret void
}

define void @test_masked_store_unaligned_v8i64(<8 x i64> %data, ptr %ptr, <8 x i1> %mask) {
; AVX-LABEL: test_masked_store_unaligned_v8i64:
; AVX:       # %bb.0:
; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]
; AVX-NEXT:    vpslld $31, %xmm3, %xmm3
; AVX-NEXT:    vpmovsxdq %xmm3, %xmm4
; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm3, %xmm3
; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX-NEXT:    vpmovsxdq %xmm2, %xmm4
; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
; AVX-NEXT:    vpmovsxdq %xmm2, %xmm2
; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
; AVX-NEXT:    vmaskmovpd %ymm0, %ymm2, 1(%rdi)
; AVX-NEXT:    vmaskmovpd %ymm1, %ymm3, 33(%rdi)
; AVX-NEXT:    vzeroupper
; AVX-NEXT:    retq
;
; AVX2-LABEL: test_masked_store_unaligned_v8i64:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]
; AVX2-NEXT:    vpslld $31, %xmm3, %xmm3
; AVX2-NEXT:    vpmovsxdq %xmm3, %ymm3
; AVX2-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
; AVX2-NEXT:    vpslld $31, %xmm2, %xmm2
; AVX2-NEXT:    vpmovsxdq %xmm2, %ymm2
; AVX2-NEXT:    vpmaskmovq %ymm0, %ymm2, 1(%rdi)
; AVX2-NEXT:    vpmaskmovq %ymm1, %ymm3, 33(%rdi)
; AVX2-NEXT:    vzeroupper
; AVX2-NEXT:    retq
;
; AVX512-LABEL: test_masked_store_unaligned_v8i64:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vpsllw $15, %xmm1, %xmm1
; AVX512-NEXT:    vpmovw2m %xmm1, %k1
; AVX512-NEXT:    vmovdqu64 %zmm0, 1(%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %ptr_i8 = getelementptr i8, ptr %ptr, i64 1
  %ptr_vec = bitcast ptr %ptr_i8 to ptr
  %load = load <8 x i64>, ptr %ptr_vec, align 1
  %sel = select <8 x i1> %mask, <8 x i64> %data, <8 x i64> %load
  store <8 x i64> %sel, ptr %ptr_vec, align 1
  ret void
}