llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=aarch64 -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE
; RUN: llc -mtriple=aarch64 -aarch64-sve-vector-bits-min=256 -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,CHECK-VL256

;; Full SVE vectors (supported with +sve)

define void @test_compressstore_nxv4i32(ptr %p, <vscale x 4 x i32> %vec, <vscale x 4 x i1> %mask) {
; CHECK-LABEL: test_compressstore_nxv4i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cntp x8, p0, p0.s
; CHECK-NEXT:    compact z0.s, p0, z0.s
; CHECK-NEXT:    whilelo p0.s, xzr, x8
; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
; CHECK-NEXT:    ret
  tail call void @llvm.masked.compressstore.nxv4i32(<vscale x 4 x i32> %vec, ptr align 4 %p, <vscale x 4 x i1> %mask)
  ret void
}

define void @test_compressstore_nxv2i64(ptr %p, <vscale x 2 x i64> %vec, <vscale x 2 x i1> %mask) {
; CHECK-LABEL: test_compressstore_nxv2i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cntp x8, p0, p0.d
; CHECK-NEXT:    compact z0.d, p0, z0.d
; CHECK-NEXT:    whilelo p0.d, xzr, x8
; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
; CHECK-NEXT:    ret
  tail call void @llvm.masked.compressstore.nxv2i64(<vscale x 2 x i64> %vec, ptr align 8 %p, <vscale x 2 x i1> %mask)
  ret void
}

define void @test_compressstore_nxv4f32(ptr %p, <vscale x 4 x float> %vec, <vscale x 4 x i1> %mask) {
; CHECK-LABEL: test_compressstore_nxv4f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cntp x8, p0, p0.s
; CHECK-NEXT:    compact z0.s, p0, z0.s
; CHECK-NEXT:    whilelo p0.s, xzr, x8
; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
; CHECK-NEXT:    ret
  tail call void @llvm.masked.compressstore.nxv4f32(<vscale x 4 x float> %vec, ptr align 4 %p, <vscale x 4 x i1> %mask)
  ret void
}

define void @test_compressstore_nxv2f64(ptr %p, <vscale x 2 x double> %vec, <vscale x 2 x i1> %mask) {
; CHECK-LABEL: test_compressstore_nxv2f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cntp x8, p0, p0.d
; CHECK-NEXT:    compact z0.d, p0, z0.d
; CHECK-NEXT:    whilelo p0.d, xzr, x8
; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
; CHECK-NEXT:    ret
  tail call void @llvm.masked.compressstore.nxv2f64(<vscale x 2 x double> %vec, ptr align 8 %p, <vscale x 2 x i1> %mask)
  ret void
}

;; SVE vectors that will be split

define void @test_compressstore_nxv8i32(ptr %p, <vscale x 8 x i32> %vec, <vscale x 8 x i1> %mask) {
; CHECK-LABEL: test_compressstore_nxv8i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    punpkhi p1.h, p0.b
; CHECK-NEXT:    punpklo p0.h, p0.b
; CHECK-NEXT:    cntp x8, p1, p1.s
; CHECK-NEXT:    compact z1.s, p1, z1.s
; CHECK-NEXT:    cntp x9, p0, p0.s
; CHECK-NEXT:    compact z0.s, p0, z0.s
; CHECK-NEXT:    whilelo p0.s, xzr, x8
; CHECK-NEXT:    whilelo p1.s, xzr, x9
; CHECK-NEXT:    st1w { z1.s }, p0, [x0, x9, lsl #2]
; CHECK-NEXT:    st1w { z0.s }, p1, [x0]
; CHECK-NEXT:    ret
  tail call void @llvm.masked.compressstore.nxv8i32(<vscale x 8 x i32> %vec, ptr align 4 %p, <vscale x 8 x i1> %mask)
  ret void
}

;; Unpacked SVE vector types

define void @test_compressstore_nxv2f32(ptr %p, <vscale x 2 x float> %vec, <vscale x 2 x i1> %mask) {
; CHECK-LABEL: test_compressstore_nxv2f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cntp x8, p0, p0.d
; CHECK-NEXT:    compact z0.d, p0, z0.d
; CHECK-NEXT:    whilelo p0.d, xzr, x8
; CHECK-NEXT:    st1w { z0.d }, p0, [x0]
; CHECK-NEXT:    ret
  tail call void @llvm.masked.compressstore.nxv2f32(<vscale x 2 x float> %vec, ptr align 4 %p, <vscale x 2 x i1> %mask)
  ret void
}

;; SVE vector types promoted to 32/64-bit (non-exhaustive)

define void @test_compressstore_nxv2i8(ptr %p, <vscale x 2 x i8> %vec, <vscale x 2 x i1> %mask) {
; CHECK-LABEL: test_compressstore_nxv2i8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cntp x8, p0, p0.d
; CHECK-NEXT:    compact z0.d, p0, z0.d
; CHECK-NEXT:    whilelo p0.d, xzr, x8
; CHECK-NEXT:    st1b { z0.d }, p0, [x0]
; CHECK-NEXT:    ret
  tail call void @llvm.masked.compressstore.nxv2i8(<vscale x 2 x i8> %vec, ptr align 1 %p, <vscale x 2 x i1> %mask)
  ret void
}

define void @test_compressstore_nxv4i16(ptr %p, <vscale x 4 x i16> %vec, <vscale x 4 x i1> %mask) {
; CHECK-LABEL: test_compressstore_nxv4i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cntp x8, p0, p0.s
; CHECK-NEXT:    compact z0.s, p0, z0.s
; CHECK-NEXT:    whilelo p0.s, xzr, x8
; CHECK-NEXT:    st1h { z0.s }, p0, [x0]
; CHECK-NEXT:    ret
  tail call void @llvm.masked.compressstore.nxv4i16(<vscale x 4 x i16> %vec, ptr align 2 %p, <vscale x 4 x i1> %mask)
  ret void
}

;; NEON vector types (promoted to SVE)

define void @test_compressstore_v2f64(ptr %p, <2 x double> %vec, <2 x i1> %mask) {
; CHECK-LABEL: test_compressstore_v2f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ushll v1.2d, v1.2s, #0
; CHECK-NEXT:    ptrue p0.d, vl2
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    shl v1.2d, v1.2d, #63
; CHECK-NEXT:    cmpne p1.d, p0/z, z1.d, #0
; CHECK-NEXT:    cntp x8, p1, p1.d
; CHECK-NEXT:    compact z0.d, p1, z0.d
; CHECK-NEXT:    whilelo p0.d, xzr, x8
; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
; CHECK-NEXT:    ret
  tail call void @llvm.masked.compressstore.v2f64(<2 x double> %vec, ptr align 8 %p, <2 x i1> %mask)
  ret void
}

define void @test_compressstore_v4i32(ptr %p, <4 x i32> %vec, <4 x i1> %mask) {
; CHECK-LABEL: test_compressstore_v4i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
; CHECK-NEXT:    ptrue p0.s, vl4
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    shl v1.4s, v1.4s, #31
; CHECK-NEXT:    cmpne p1.s, p0/z, z1.s, #0
; CHECK-NEXT:    cntp x8, p1, p1.s
; CHECK-NEXT:    compact z0.s, p1, z0.s
; CHECK-NEXT:    whilelo p0.s, xzr, x8
; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
; CHECK-NEXT:    ret
  tail call void @llvm.masked.compressstore.v4i32(<4 x i32> %vec, ptr align 4 %p, <4 x i1> %mask)
  ret void
}

define void @test_compressstore_v2i64(ptr %p, <2 x i64> %vec, <2 x i1> %mask) {
; CHECK-LABEL: test_compressstore_v2i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ushll v1.2d, v1.2s, #0
; CHECK-NEXT:    ptrue p0.d, vl2
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    shl v1.2d, v1.2d, #63
; CHECK-NEXT:    cmpne p1.d, p0/z, z1.d, #0
; CHECK-NEXT:    cntp x8, p1, p1.d
; CHECK-NEXT:    compact z0.d, p1, z0.d
; CHECK-NEXT:    whilelo p0.d, xzr, x8
; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
; CHECK-NEXT:    ret
  tail call void @llvm.masked.compressstore.v2i64(<2 x i64> %vec, ptr align 8 %p, <2 x i1> %mask)
  ret void
}

define void @test_compressstore_v8i32(ptr %p, <8 x i32> %vec, <8 x i1> %mask) {
; CHECK-BASE-LABEL: test_compressstore_v8i32:
; CHECK-BASE:       // %bb.0:
; CHECK-BASE-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-BASE-NEXT:    zip1 v3.8b, v2.8b, v0.8b
; CHECK-BASE-NEXT:    adrp x8, .LCPI11_0
; CHECK-BASE-NEXT:    zip2 v2.8b, v2.8b, v0.8b
; CHECK-BASE-NEXT:    ldr d5, [x8, :lo12:.LCPI11_0]
; CHECK-BASE-NEXT:    ptrue p0.s
; CHECK-BASE-NEXT:    // kill: def $q1 killed $q1 def $z1
; CHECK-BASE-NEXT:    ptrue p1.s, vl4
; CHECK-BASE-NEXT:    shl v4.4h, v3.4h, #15
; CHECK-BASE-NEXT:    ushll v2.4s, v2.4h, #0
; CHECK-BASE-NEXT:    ushll v3.4s, v3.4h, #0
; CHECK-BASE-NEXT:    cmlt v4.4h, v4.4h, #0
; CHECK-BASE-NEXT:    shl v2.4s, v2.4s, #31
; CHECK-BASE-NEXT:    shl v3.4s, v3.4s, #31
; CHECK-BASE-NEXT:    and v4.8b, v4.8b, v5.8b
; CHECK-BASE-NEXT:    cmpne p2.s, p1/z, z3.s, #0
; CHECK-BASE-NEXT:    addv h4, v4.4h
; CHECK-BASE-NEXT:    cntp x9, p2, p2.s
; CHECK-BASE-NEXT:    compact z0.s, p2, z0.s
; CHECK-BASE-NEXT:    fmov w8, s4
; CHECK-BASE-NEXT:    and w8, w8, #0xf
; CHECK-BASE-NEXT:    fmov s4, w8
; CHECK-BASE-NEXT:    cnt z4.s, p0/m, z4.s
; CHECK-BASE-NEXT:    cmpne p0.s, p1/z, z2.s, #0
; CHECK-BASE-NEXT:    whilelo p1.s, xzr, x9
; CHECK-BASE-NEXT:    cntp x8, p0, p0.s
; CHECK-BASE-NEXT:    compact z1.s, p0, z1.s
; CHECK-BASE-NEXT:    fmov w10, s4
; CHECK-BASE-NEXT:    whilelo p0.s, xzr, x8
; CHECK-BASE-NEXT:    st1w { z1.s }, p0, [x0, x10, lsl #2]
; CHECK-BASE-NEXT:    st1w { z0.s }, p1, [x0]
; CHECK-BASE-NEXT:    ret
;
; CHECK-VL256-LABEL: test_compressstore_v8i32:
; CHECK-VL256:       // %bb.0:
; CHECK-VL256-NEXT:    // kill: def $d2 killed $d2 def $z2
; CHECK-VL256-NEXT:    ptrue p0.s, vl8
; CHECK-VL256-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-VL256-NEXT:    // kill: def $q1 killed $q1 def $z1
; CHECK-VL256-NEXT:    uunpklo z2.h, z2.b
; CHECK-VL256-NEXT:    ptrue p1.s, vl4
; CHECK-VL256-NEXT:    splice z0.s, p1, z0.s, z1.s
; CHECK-VL256-NEXT:    uunpklo z2.s, z2.h
; CHECK-VL256-NEXT:    lsl z2.s, z2.s, #31
; CHECK-VL256-NEXT:    asr z2.s, z2.s, #31
; CHECK-VL256-NEXT:    cmpne p2.s, p0/z, z2.s, #0
; CHECK-VL256-NEXT:    cntp x8, p2, p2.s
; CHECK-VL256-NEXT:    compact z0.s, p2, z0.s
; CHECK-VL256-NEXT:    whilelo p0.s, xzr, x8
; CHECK-VL256-NEXT:    st1w { z0.s }, p0, [x0]
; CHECK-VL256-NEXT:    ret
  tail call void @llvm.masked.compressstore.v8i32(<8 x i32> %vec, ptr align 4 %p, <8 x i1> %mask)
  ret void
}

define void @test_compressstore_v4i64(ptr %p, <4 x i64> %vec, <4 x i1> %mask) {
; CHECK-BASE-LABEL: test_compressstore_v4i64:
; CHECK-BASE:       // %bb.0:
; CHECK-BASE-NEXT:    ushll v2.4s, v2.4h, #0
; CHECK-BASE-NEXT:    index z4.s, #1, #1
; CHECK-BASE-NEXT:    ptrue p0.s
; CHECK-BASE-NEXT:    ptrue p1.d, vl2
; CHECK-BASE-NEXT:    // kill: def $q1 killed $q1 def $z1
; CHECK-BASE-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-BASE-NEXT:    shl v3.2s, v2.2s, #31
; CHECK-BASE-NEXT:    cmlt v3.2s, v3.2s, #0
; CHECK-BASE-NEXT:    and v3.8b, v3.8b, v4.8b
; CHECK-BASE-NEXT:    ushll2 v4.2d, v2.4s, #0
; CHECK-BASE-NEXT:    ushll v2.2d, v2.2s, #0
; CHECK-BASE-NEXT:    addp v3.2s, v3.2s, v3.2s
; CHECK-BASE-NEXT:    shl v2.2d, v2.2d, #63
; CHECK-BASE-NEXT:    fmov w8, s3
; CHECK-BASE-NEXT:    shl v3.2d, v4.2d, #63
; CHECK-BASE-NEXT:    cmpne p2.d, p1/z, z2.d, #0
; CHECK-BASE-NEXT:    and w8, w8, #0x3
; CHECK-BASE-NEXT:    fmov s4, w8
; CHECK-BASE-NEXT:    cntp x9, p2, p2.d
; CHECK-BASE-NEXT:    compact z0.d, p2, z0.d
; CHECK-BASE-NEXT:    cnt z4.s, p0/m, z4.s
; CHECK-BASE-NEXT:    cmpne p0.d, p1/z, z3.d, #0
; CHECK-BASE-NEXT:    whilelo p1.d, xzr, x9
; CHECK-BASE-NEXT:    cntp x8, p0, p0.d
; CHECK-BASE-NEXT:    compact z1.d, p0, z1.d
; CHECK-BASE-NEXT:    fmov w10, s4
; CHECK-BASE-NEXT:    whilelo p0.d, xzr, x8
; CHECK-BASE-NEXT:    st1d { z1.d }, p0, [x0, x10, lsl #3]
; CHECK-BASE-NEXT:    st1d { z0.d }, p1, [x0]
; CHECK-BASE-NEXT:    ret
;
; CHECK-VL256-LABEL: test_compressstore_v4i64:
; CHECK-VL256:       // %bb.0:
; CHECK-VL256-NEXT:    // kill: def $d2 killed $d2 def $z2
; CHECK-VL256-NEXT:    ptrue p0.d, vl4
; CHECK-VL256-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-VL256-NEXT:    // kill: def $q1 killed $q1 def $z1
; CHECK-VL256-NEXT:    uunpklo z2.s, z2.h
; CHECK-VL256-NEXT:    ptrue p1.d, vl2
; CHECK-VL256-NEXT:    splice z0.d, p1, z0.d, z1.d
; CHECK-VL256-NEXT:    uunpklo z2.d, z2.s
; CHECK-VL256-NEXT:    lsl z2.d, z2.d, #63
; CHECK-VL256-NEXT:    asr z2.d, z2.d, #63
; CHECK-VL256-NEXT:    cmpne p2.d, p0/z, z2.d, #0
; CHECK-VL256-NEXT:    cntp x8, p2, p2.d
; CHECK-VL256-NEXT:    compact z0.d, p2, z0.d
; CHECK-VL256-NEXT:    whilelo p0.d, xzr, x8
; CHECK-VL256-NEXT:    st1d { z0.d }, p0, [x0]
; CHECK-VL256-NEXT:    ret
  tail call void @llvm.masked.compressstore.v4i64(<4 x i64> %vec, ptr align 8 %p, <4 x i1> %mask)
  ret void
}