test/CodeGen/X86/avx512-masked-memop-64-32.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=x86_64-apple-darwin -mattr=avx512f < %s | FileCheck %s --check-prefix=AVX512 --check-prefix=AVX512F
; RUN: llc -mtriple=x86_64-apple-darwin -mattr=avx512f,avx512bw,avx512vl < %s | FileCheck %s --check-prefix=AVX512 --check-prefix=SKX

define <16 x i32> @test1(<16 x i32> %trigger, ptr %addr) {
; AVX512-LABEL: test1:
; AVX512:       ## %bb.0:
; AVX512-NEXT:    vptestnmd %zmm0, %zmm0, %k1
; AVX512-NEXT:    vmovdqu32 (%rdi), %zmm0 {%k1} {z}
; AVX512-NEXT:    retq
  %mask = icmp eq <16 x i32> %trigger, zeroinitializer
  %res = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr %addr, i32 4, <16 x i1>%mask, <16 x i32>undef)
  ret <16 x i32> %res
}

define <16 x i32> @test2(<16 x i32> %trigger, ptr %addr) {
; AVX512-LABEL: test2:
; AVX512:       ## %bb.0:
; AVX512-NEXT:    vptestnmd %zmm0, %zmm0, %k1
; AVX512-NEXT:    vmovdqu32 (%rdi), %zmm0 {%k1} {z}
; AVX512-NEXT:    retq
  %mask = icmp eq <16 x i32> %trigger, zeroinitializer
  %res = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr %addr, i32 4, <16 x i1>%mask, <16 x i32>zeroinitializer)
  ret <16 x i32> %res
}

define void @test3(<16 x i32> %trigger, ptr %addr, <16 x i32> %val) {
; AVX512-LABEL: test3:
; AVX512:       ## %bb.0:
; AVX512-NEXT:    vptestnmd %zmm0, %zmm0, %k1
; AVX512-NEXT:    vmovdqu32 %zmm1, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %mask = icmp eq <16 x i32> %trigger, zeroinitializer
  call void @llvm.masked.store.v16i32.p0(<16 x i32>%val, ptr %addr, i32 4, <16 x i1>%mask)
  ret void
}

define <16 x float> @test4(<16 x i32> %trigger, ptr %addr, <16 x float> %dst) {
; AVX512-LABEL: test4:
; AVX512:       ## %bb.0:
; AVX512-NEXT:    vptestnmd %zmm0, %zmm0, %k1
; AVX512-NEXT:    vblendmps (%rdi), %zmm1, %zmm0 {%k1}
; AVX512-NEXT:    retq
  %mask = icmp eq <16 x i32> %trigger, zeroinitializer
  %res = call <16 x float> @llvm.masked.load.v16f32.p0(ptr %addr, i32 4, <16 x i1>%mask, <16 x float> %dst)
  ret <16 x float> %res
}

define void @test13(<16 x i32> %trigger, ptr %addr, <16 x float> %val) {
; AVX512-LABEL: test13:
; AVX512:       ## %bb.0:
; AVX512-NEXT:    vptestnmd %zmm0, %zmm0, %k1
; AVX512-NEXT:    vmovups %zmm1, (%rdi) {%k1}
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  %mask = icmp eq <16 x i32> %trigger, zeroinitializer
  call void @llvm.masked.store.v16f32.p0(<16 x float>%val, ptr %addr, i32 4, <16 x i1>%mask)
  ret void
}

define void @one_mask_bit_set5(ptr %addr, <8 x double> %val) {
; AVX512-LABEL: one_mask_bit_set5:
; AVX512:       ## %bb.0:
; AVX512-NEXT:    vextractf32x4 $3, %zmm0, %xmm0
; AVX512-NEXT:    vmovlps %xmm0, 48(%rdi)
; AVX512-NEXT:    vzeroupper
; AVX512-NEXT:    retq
  call void @llvm.masked.store.v8f64.p0(<8 x double> %val, ptr %addr, i32 4, <8 x i1><i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false>)
  ret void
}

define <8 x double> @load_one_mask_bit_set5(ptr %addr, <8 x double> %val) {
;
; AVX512F-LABEL: load_one_mask_bit_set5:
; AVX512F:       ## %bb.0:
; AVX512F-NEXT:    movb $-128, %al
; AVX512F-NEXT:    kmovw %eax, %k1
; AVX512F-NEXT:    vbroadcastsd 56(%rdi), %zmm0 {%k1}
; AVX512F-NEXT:    retq
;
; SKX-LABEL: load_one_mask_bit_set5:
; SKX:       ## %bb.0:
; SKX-NEXT:    movb $-128, %al
; SKX-NEXT:    kmovd %eax, %k1
; SKX-NEXT:    vbroadcastsd 56(%rdi), %zmm0 {%k1}
; SKX-NEXT:    retq
  %res = call <8 x double> @llvm.masked.load.v8f64.p0(ptr %addr, i32 4, <8 x i1><i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <8 x double> %val)
  ret <8 x double> %res
}

declare <16 x i32> @llvm.masked.load.v16i32.p0(ptr, i32, <16 x i1>, <16 x i32>)
declare void @llvm.masked.store.v16i32.p0(<16 x i32>, ptr, i32, <16 x i1>)
declare void @llvm.masked.store.v16f32.p0(<16 x float>, ptr, i32, <16 x i1>)
declare <16 x float> @llvm.masked.load.v16f32.p0(ptr, i32, <16 x i1>, <16 x float>)
declare <8 x double> @llvm.masked.load.v8f64.p0(ptr, i32, <8 x i1>, <8 x double>)
declare void @llvm.masked.store.v8f64.p0(<8 x double>, ptr, i32, <8 x i1>)

declare <16 x ptr> @llvm.masked.load.v16p0.p0(ptr, i32, <16 x i1>, <16 x ptr>)

define <16 x ptr> @test23(<16 x ptr> %trigger, ptr %addr) {
; AVX512-LABEL: test23:
; AVX512:       ## %bb.0:
; AVX512-NEXT:    vptestnmq %zmm1, %zmm1, %k1
; AVX512-NEXT:    vptestnmq %zmm0, %zmm0, %k2
; AVX512-NEXT:    vmovdqu64 (%rdi), %zmm0 {%k2} {z}
; AVX512-NEXT:    vmovdqu64 64(%rdi), %zmm1 {%k1} {z}
; AVX512-NEXT:    retq
  %mask = icmp eq <16 x ptr> %trigger, zeroinitializer
  %res = call <16 x ptr> @llvm.masked.load.v16p0.p0(ptr %addr, i32 4, <16 x i1>%mask, <16 x ptr>zeroinitializer)
  ret <16 x ptr> %res
}

%mystruct = type { i16, i16, [1 x ptr] }


define <16 x ptr> @test24(<16 x i1> %mask, ptr %addr) {
; AVX512F-LABEL: test24:
; AVX512F:       ## %bb.0:
; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
; AVX512F-NEXT:    vmovdqu64 (%rdi), %zmm0 {%k1} {z}
; AVX512F-NEXT:    kshiftrw $8, %k1, %k1
; AVX512F-NEXT:    vmovdqu64 64(%rdi), %zmm1 {%k1} {z}
; AVX512F-NEXT:    retq
;
; SKX-LABEL: test24:
; SKX:       ## %bb.0:
; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0
; SKX-NEXT:    vpmovb2m %xmm0, %k1
; SKX-NEXT:    vmovdqu64 (%rdi), %zmm0 {%k1} {z}
; SKX-NEXT:    kshiftrw $8, %k1, %k1
; SKX-NEXT:    vmovdqu64 64(%rdi), %zmm1 {%k1} {z}
; SKX-NEXT:    retq
  %res = call <16 x ptr> @llvm.masked.load.v16p0.p0(ptr %addr, i32 4, <16 x i1>%mask, <16 x ptr>zeroinitializer)
  ret <16 x ptr> %res
}

define void @test_store_16i64(ptr %ptrs, <16 x i1> %mask, <16 x i64> %src0)  {
; AVX512F-LABEL: test_store_16i64:
; AVX512F:       ## %bb.0:
; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
; AVX512F-NEXT:    vmovdqu64 %zmm1, (%rdi) {%k1}
; AVX512F-NEXT:    kshiftrw $8, %k1, %k1
; AVX512F-NEXT:    vmovdqu64 %zmm2, 64(%rdi) {%k1}
; AVX512F-NEXT:    vzeroupper
; AVX512F-NEXT:    retq
;
; SKX-LABEL: test_store_16i64:
; SKX:       ## %bb.0:
; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0
; SKX-NEXT:    vpmovb2m %xmm0, %k1
; SKX-NEXT:    vmovdqu64 %zmm1, (%rdi) {%k1}
; SKX-NEXT:    kshiftrw $8, %k1, %k1
; SKX-NEXT:    vmovdqu64 %zmm2, 64(%rdi) {%k1}
; SKX-NEXT:    vzeroupper
; SKX-NEXT:    retq
  call void @llvm.masked.store.v16i64.p0(<16 x i64> %src0, ptr %ptrs, i32 4, <16 x i1> %mask)
  ret void
}
declare void @llvm.masked.store.v16i64.p0(<16 x i64> %src0, ptr %ptrs, i32, <16 x i1> %mask)

define void @test_store_16f64(ptr %ptrs, <16 x i1> %mask, <16 x double> %src0)  {
; AVX512F-LABEL: test_store_16f64:
; AVX512F:       ## %bb.0:
; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
; AVX512F-NEXT:    vmovupd %zmm1, (%rdi) {%k1}
; AVX512F-NEXT:    kshiftrw $8, %k1, %k1
; AVX512F-NEXT:    vmovupd %zmm2, 64(%rdi) {%k1}
; AVX512F-NEXT:    vzeroupper
; AVX512F-NEXT:    retq
;
; SKX-LABEL: test_store_16f64:
; SKX:       ## %bb.0:
; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0
; SKX-NEXT:    vpmovb2m %xmm0, %k1
; SKX-NEXT:    vmovupd %zmm1, (%rdi) {%k1}
; SKX-NEXT:    kshiftrw $8, %k1, %k1
; SKX-NEXT:    vmovupd %zmm2, 64(%rdi) {%k1}
; SKX-NEXT:    vzeroupper
; SKX-NEXT:    retq
  call void @llvm.masked.store.v16f64.p0(<16 x double> %src0, ptr %ptrs, i32 4, <16 x i1> %mask)
  ret void
}
declare void @llvm.masked.store.v16f64.p0(<16 x double> %src0, ptr %ptrs, i32, <16 x i1> %mask)

define <16 x i64> @test_load_16i64(ptr %ptrs, <16 x i1> %mask, <16 x i64> %src0)  {
; AVX512F-LABEL: test_load_16i64:
; AVX512F:       ## %bb.0:
; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
; AVX512F-NEXT:    vpblendmq (%rdi), %zmm1, %zmm0 {%k1}
; AVX512F-NEXT:    kshiftrw $8, %k1, %k1
; AVX512F-NEXT:    vpblendmq 64(%rdi), %zmm2, %zmm1 {%k1}
; AVX512F-NEXT:    retq
;
; SKX-LABEL: test_load_16i64:
; SKX:       ## %bb.0:
; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0
; SKX-NEXT:    vpmovb2m %xmm0, %k1
; SKX-NEXT:    vpblendmq (%rdi), %zmm1, %zmm0 {%k1}
; SKX-NEXT:    kshiftrw $8, %k1, %k1
; SKX-NEXT:    vpblendmq 64(%rdi), %zmm2, %zmm1 {%k1}
; SKX-NEXT:    retq
  %res = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr %ptrs, i32 4, <16 x i1> %mask, <16 x i64> %src0)
  ret <16 x i64> %res
}
declare <16 x i64> @llvm.masked.load.v16i64.p0(ptr %ptrs, i32, <16 x i1> %mask, <16 x i64> %src0)

define <16 x double> @test_load_16f64(ptr %ptrs, <16 x i1> %mask, <16 x double> %src0)  {
; AVX512F-LABEL: test_load_16f64:
; AVX512F:       ## %bb.0:
; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
; AVX512F-NEXT:    vblendmpd (%rdi), %zmm1, %zmm0 {%k1}
; AVX512F-NEXT:    kshiftrw $8, %k1, %k1
; AVX512F-NEXT:    vblendmpd 64(%rdi), %zmm2, %zmm1 {%k1}
; AVX512F-NEXT:    retq
;
; SKX-LABEL: test_load_16f64:
; SKX:       ## %bb.0:
; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0
; SKX-NEXT:    vpmovb2m %xmm0, %k1
; SKX-NEXT:    vblendmpd (%rdi), %zmm1, %zmm0 {%k1}
; SKX-NEXT:    kshiftrw $8, %k1, %k1
; SKX-NEXT:    vblendmpd 64(%rdi), %zmm2, %zmm1 {%k1}
; SKX-NEXT:    retq
  %res = call <16 x double> @llvm.masked.load.v16f64.p0(ptr %ptrs, i32 4, <16 x i1> %mask, <16 x double> %src0)
  ret <16 x double> %res
}
declare <16 x double> @llvm.masked.load.v16f64.p0(ptr %ptrs, i32, <16 x i1> %mask, <16 x double> %src0)

define <32 x double> @test_load_32f64(ptr %ptrs, <32 x i1> %mask, <32 x double> %src0)  {
; AVX512F-LABEL: test_load_32f64:
; AVX512F:       ## %bb.0:
; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm5
; AVX512F-NEXT:    vpmovsxbd %xmm5, %zmm5
; AVX512F-NEXT:    vpslld $31, %zmm5, %zmm5
; AVX512F-NEXT:    vptestmd %zmm5, %zmm5, %k1
; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k2
; AVX512F-NEXT:    vblendmpd (%rdi), %zmm1, %zmm0 {%k2}
; AVX512F-NEXT:    vblendmpd 128(%rdi), %zmm3, %zmm5 {%k1}
; AVX512F-NEXT:    kshiftrw $8, %k2, %k2
; AVX512F-NEXT:    vblendmpd 64(%rdi), %zmm2, %zmm1 {%k2}
; AVX512F-NEXT:    kshiftrw $8, %k1, %k1
; AVX512F-NEXT:    vblendmpd 192(%rdi), %zmm4, %zmm3 {%k1}
; AVX512F-NEXT:    vmovapd %zmm5, %zmm2
; AVX512F-NEXT:    retq
;
; SKX-LABEL: test_load_32f64:
; SKX:       ## %bb.0:
; SKX-NEXT:    vpsllw $7, %ymm0, %ymm0
; SKX-NEXT:    vpmovb2m %ymm0, %k1
; SKX-NEXT:    vblendmpd (%rdi), %zmm1, %zmm0 {%k1}
; SKX-NEXT:    kshiftrw $8, %k1, %k2
; SKX-NEXT:    vblendmpd 64(%rdi), %zmm2, %zmm1 {%k2}
; SKX-NEXT:    kshiftrd $16, %k1, %k1
; SKX-NEXT:    vblendmpd 128(%rdi), %zmm3, %zmm2 {%k1}
; SKX-NEXT:    kshiftrw $8, %k1, %k1
; SKX-NEXT:    vblendmpd 192(%rdi), %zmm4, %zmm3 {%k1}
; SKX-NEXT:    retq
  %res = call <32 x double> @llvm.masked.load.v32f64.p0(ptr %ptrs, i32 4, <32 x i1> %mask, <32 x double> %src0)
  ret <32 x double> %res
}

declare <32 x double> @llvm.masked.load.v32f64.p0(ptr %ptrs, i32, <32 x i1> %mask, <32 x double> %src0)