llvm/test/CodeGen/AArch64/sve-mask-partition.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64 -mattr=+sve < %s | FileCheck %s

;; Scalable
define <vscale x 16 x i1> @mask_exclude_active_nxv16(<vscale x 16 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_nxv16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.b
; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> %mask.in, i1 false)
  %mask.out = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 %tz.elts)
  ret <vscale x 16 x i1> %mask.out
}

define <vscale x 8 x i1> @mask_exclude_active_nxv8(<vscale x 8 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_nxv8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.h
; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> %mask.in, i1 false)
  %mask.out = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 %tz.elts)
  ret <vscale x 8 x i1> %mask.out
}

define <vscale x 4 x i1> @mask_exclude_active_nxv4(<vscale x 4 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_nxv4:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.s
; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> %mask.in, i1 false)
  %mask.out = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 %tz.elts)
  ret <vscale x 4 x i1> %mask.out
}

define <vscale x 2 x i1> @mask_exclude_active_nxv2(<vscale x 2 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_nxv2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.d
; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> %mask.in, i1 false)
  %mask.out = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 %tz.elts)
  ret <vscale x 2 x i1> %mask.out
}

define <vscale x 16 x i1> @mask_include_active_nxv16(<vscale x 16 x i1> %mask.in) {
; CHECK-LABEL: mask_include_active_nxv16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.b
; CHECK-NEXT:    brka p0.b, p1/z, p0.b
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> %mask.in, i1 false)
  %inc = add i64 %tz.elts, 1
  %mask.out = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 %inc)
  ret <vscale x 16 x i1> %mask.out
}

define <vscale x 8 x i1> @mask_include_active_nxv8(<vscale x 8 x i1> %mask.in) {
; CHECK-LABEL: mask_include_active_nxv8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.h
; CHECK-NEXT:    brka p0.b, p1/z, p0.b
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> %mask.in, i1 false)
  %inc = add i64 %tz.elts, 1
  %mask.out = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 %inc)
  ret <vscale x 8 x i1> %mask.out
}

define <vscale x 4 x i1> @mask_include_active_nxv4(<vscale x 4 x i1> %mask.in) {
; CHECK-LABEL: mask_include_active_nxv4:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.s
; CHECK-NEXT:    brka p0.b, p1/z, p0.b
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> %mask.in, i1 false)
  %inc = add i64 %tz.elts, 1
  %mask.out = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 %inc)
  ret <vscale x 4 x i1> %mask.out
}

define <vscale x 2 x i1> @mask_include_active_nxv2(<vscale x 2 x i1> %mask.in) {
; CHECK-LABEL: mask_include_active_nxv2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.d
; CHECK-NEXT:    brka p0.b, p1/z, p0.b
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> %mask.in, i1 false)
  %inc = add i64 %tz.elts, 1
  %mask.out = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 %inc)
  ret <vscale x 2 x i1> %mask.out
}

;; Fixed
define <16 x i1> @mask_exclude_active_v16(<16 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_v16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shl v0.16b, v0.16b, #7
; CHECK-NEXT:    ptrue p0.b, vl16
; CHECK-NEXT:    cmpne p1.b, p0/z, z0.b, #0
; CHECK-NEXT:    brkb p1.b, p0/z, p1.b
; CHECK-NEXT:    mov z0.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> %mask.in, i1 false)
  %mask.out = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 %tz.elts)
  ret <16 x i1> %mask.out
}

define <8 x i1> @mask_exclude_active_v8(<8 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_v8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shl v0.8b, v0.8b, #7
; CHECK-NEXT:    ptrue p0.b, vl8
; CHECK-NEXT:    cmpne p1.b, p0/z, z0.b, #0
; CHECK-NEXT:    brkb p1.b, p0/z, p1.b
; CHECK-NEXT:    mov z0.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> %mask.in, i1 false)
  %mask.out = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 %tz.elts)
  ret <8 x i1> %mask.out
}

define <4 x i1> @mask_exclude_active_v4(<4 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_v4:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shl v0.4h, v0.4h, #15
; CHECK-NEXT:    ptrue p0.h, vl4
; CHECK-NEXT:    cmpne p1.h, p0/z, z0.h, #0
; CHECK-NEXT:    brkb p1.b, p0/z, p1.b
; CHECK-NEXT:    mov z0.h, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> %mask.in, i1 false)
  %mask.out = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 %tz.elts)
  ret <4 x i1> %mask.out
}

define <2 x i1> @mask_exclude_active_v2(<2 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_v2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shl v0.2s, v0.2s, #31
; CHECK-NEXT:    ptrue p0.s, vl2
; CHECK-NEXT:    cmpne p1.s, p0/z, z0.s, #0
; CHECK-NEXT:    brkb p1.b, p0/z, p1.b
; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> %mask.in, i1 false)
  %mask.out = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 %tz.elts)
  ret <2 x i1> %mask.out
}

define <16 x i1> @mask_include_active_v16(<16 x i1> %mask.in) {
; CHECK-LABEL: mask_include_active_v16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shl v0.16b, v0.16b, #7
; CHECK-NEXT:    ptrue p0.b, vl16
; CHECK-NEXT:    cmpne p1.b, p0/z, z0.b, #0
; CHECK-NEXT:    brka p1.b, p0/z, p1.b
; CHECK-NEXT:    mov z0.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> %mask.in, i1 false)
  %inc = add i64 %tz.elts, 1
  %mask.out = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 %inc)
  ret <16 x i1> %mask.out
}

define <8 x i1> @mask_include_active_v8(<8 x i1> %mask.in) {
; CHECK-LABEL: mask_include_active_v8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shl v0.8b, v0.8b, #7
; CHECK-NEXT:    ptrue p0.b, vl8
; CHECK-NEXT:    cmpne p1.b, p0/z, z0.b, #0
; CHECK-NEXT:    brka p1.b, p0/z, p1.b
; CHECK-NEXT:    mov z0.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> %mask.in, i1 false)
  %inc = add i64 %tz.elts, 1
  %mask.out = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 %inc)
  ret <8 x i1> %mask.out
}

define <4 x i1> @mask_include_active_v4(<4 x i1> %mask.in) {
; CHECK-LABEL: mask_include_active_v4:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shl v0.4h, v0.4h, #15
; CHECK-NEXT:    ptrue p0.h, vl4
; CHECK-NEXT:    cmpne p1.h, p0/z, z0.h, #0
; CHECK-NEXT:    brka p1.b, p0/z, p1.b
; CHECK-NEXT:    mov z0.h, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> %mask.in, i1 false)
  %inc = add i64 %tz.elts, 1
  %mask.out = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 %inc)
  ret <4 x i1> %mask.out
}

define <2 x i1> @mask_include_active_v2(<2 x i1> %mask.in) {
; CHECK-LABEL: mask_include_active_v2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shl v0.2s, v0.2s, #31
; CHECK-NEXT:    ptrue p0.s, vl2
; CHECK-NEXT:    cmpne p1.s, p0/z, z0.s, #0
; CHECK-NEXT:    brka p1.b, p0/z, p1.b
; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> %mask.in, i1 false)
  %inc = add i64 %tz.elts, 1
  %mask.out = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 %inc)
  ret <2 x i1> %mask.out
}

;; Wider-than-legal tests
define <vscale x 32 x i1> @mask_exclude_active_nxv32(<vscale x 32 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_nxv32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p2.b
; CHECK-NEXT:    rdvl x8, #1
; CHECK-NEXT:    mov x9, x8
; CHECK-NEXT:    brkb p0.b, p2/z, p0.b
; CHECK-NEXT:    brkb p1.b, p2/z, p1.b
; CHECK-NEXT:    cntp x10, p0, p0.b
; CHECK-NEXT:    incp x9, p1.b
; CHECK-NEXT:    cmp x10, x8
; CHECK-NEXT:    csel x9, x10, x9, ne
; CHECK-NEXT:    whilelo p0.b, xzr, x9
; CHECK-NEXT:    whilelo p1.b, x8, x9
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> %mask.in, i1 false)
  %mask.out = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i64(i64 0, i64 %tz.elts)
  ret <vscale x 32 x i1> %mask.out
}

define <32 x i1> @mask_exclude_active_v32(<32 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_v32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr w9, [sp, #64]
; CHECK-NEXT:    fmov s0, w0
; CHECK-NEXT:    ldr w10, [sp, #72]
; CHECK-NEXT:    ptrue p0.b, vl16
; CHECK-NEXT:    fmov s1, w9
; CHECK-NEXT:    ldr w9, [sp, #80]
; CHECK-NEXT:    mov v0.b[1], w1
; CHECK-NEXT:    mov v1.b[1], w10
; CHECK-NEXT:    ldr w10, [sp, #128]
; CHECK-NEXT:    mov v0.b[2], w2
; CHECK-NEXT:    mov v1.b[2], w9
; CHECK-NEXT:    ldr w9, [sp, #88]
; CHECK-NEXT:    mov v0.b[3], w3
; CHECK-NEXT:    mov v1.b[3], w9
; CHECK-NEXT:    ldr w9, [sp, #96]
; CHECK-NEXT:    mov v0.b[4], w4
; CHECK-NEXT:    mov v1.b[4], w9
; CHECK-NEXT:    ldr w9, [sp, #104]
; CHECK-NEXT:    mov v0.b[5], w5
; CHECK-NEXT:    mov v1.b[5], w9
; CHECK-NEXT:    ldr w9, [sp, #112]
; CHECK-NEXT:    mov v0.b[6], w6
; CHECK-NEXT:    mov v1.b[6], w9
; CHECK-NEXT:    ldr w9, [sp, #120]
; CHECK-NEXT:    mov v0.b[7], w7
; CHECK-NEXT:    mov v1.b[7], w9
; CHECK-NEXT:    ldr w9, [sp]
; CHECK-NEXT:    mov v0.b[8], w9
; CHECK-NEXT:    ldr w9, [sp, #8]
; CHECK-NEXT:    mov v1.b[8], w10
; CHECK-NEXT:    ldr w10, [sp, #136]
; CHECK-NEXT:    mov v0.b[9], w9
; CHECK-NEXT:    ldr w9, [sp, #16]
; CHECK-NEXT:    mov v1.b[9], w10
; CHECK-NEXT:    ldr w10, [sp, #144]
; CHECK-NEXT:    mov v0.b[10], w9
; CHECK-NEXT:    ldr w9, [sp, #24]
; CHECK-NEXT:    mov v1.b[10], w10
; CHECK-NEXT:    ldr w10, [sp, #152]
; CHECK-NEXT:    mov v0.b[11], w9
; CHECK-NEXT:    ldr w9, [sp, #32]
; CHECK-NEXT:    mov v1.b[11], w10
; CHECK-NEXT:    ldr w10, [sp, #160]
; CHECK-NEXT:    mov v0.b[12], w9
; CHECK-NEXT:    ldr w9, [sp, #40]
; CHECK-NEXT:    mov v1.b[12], w10
; CHECK-NEXT:    ldr w10, [sp, #168]
; CHECK-NEXT:    mov v0.b[13], w9
; CHECK-NEXT:    ldr w9, [sp, #48]
; CHECK-NEXT:    mov v1.b[13], w10
; CHECK-NEXT:    ldr w10, [sp, #176]
; CHECK-NEXT:    mov v0.b[14], w9
; CHECK-NEXT:    ldr w9, [sp, #56]
; CHECK-NEXT:    mov v1.b[14], w10
; CHECK-NEXT:    ldr w10, [sp, #184]
; CHECK-NEXT:    mov v0.b[15], w9
; CHECK-NEXT:    mov v1.b[15], w10
; CHECK-NEXT:    shl v0.16b, v0.16b, #7
; CHECK-NEXT:    shl v1.16b, v1.16b, #7
; CHECK-NEXT:    cmpne p1.b, p0/z, z0.b, #0
; CHECK-NEXT:    index z0.d, #0, #1
; CHECK-NEXT:    cmpne p2.b, p0/z, z1.b, #0
; CHECK-NEXT:    brkb p1.b, p0/z, p1.b
; CHECK-NEXT:    movprfx z1, z0
; CHECK-NEXT:    add z1.d, z1.d, #14 // =0xe
; CHECK-NEXT:    movprfx z2, z0
; CHECK-NEXT:    add z2.d, z2.d, #12 // =0xc
; CHECK-NEXT:    movprfx z3, z0
; CHECK-NEXT:    add z3.d, z3.d, #10 // =0xa
; CHECK-NEXT:    movprfx z4, z0
; CHECK-NEXT:    add z4.d, z4.d, #8 // =0x8
; CHECK-NEXT:    movprfx z5, z0
; CHECK-NEXT:    add z5.d, z5.d, #6 // =0x6
; CHECK-NEXT:    brkb p2.b, p0/z, p2.b
; CHECK-NEXT:    movprfx z6, z0
; CHECK-NEXT:    add z6.d, z6.d, #4 // =0x4
; CHECK-NEXT:    movprfx z7, z0
; CHECK-NEXT:    add z7.d, z7.d, #2 // =0x2
; CHECK-NEXT:    cntp x9, p1, p1.b
; CHECK-NEXT:    movprfx z17, z0
; CHECK-NEXT:    add z17.d, z17.d, #30 // =0x1e
; CHECK-NEXT:    movprfx z18, z0
; CHECK-NEXT:    add z18.d, z18.d, #28 // =0x1c
; CHECK-NEXT:    cntp x10, p2, p2.b
; CHECK-NEXT:    movprfx z19, z0
; CHECK-NEXT:    add z19.d, z19.d, #26 // =0x1a
; CHECK-NEXT:    movprfx z20, z0
; CHECK-NEXT:    add z20.d, z20.d, #24 // =0x18
; CHECK-NEXT:    movprfx z21, z0
; CHECK-NEXT:    add z21.d, z21.d, #22 // =0x16
; CHECK-NEXT:    movprfx z22, z0
; CHECK-NEXT:    add z22.d, z22.d, #20 // =0x14
; CHECK-NEXT:    movprfx z23, z0
; CHECK-NEXT:    add z23.d, z23.d, #18 // =0x12
; CHECK-NEXT:    movprfx z24, z0
; CHECK-NEXT:    add z24.d, z24.d, #16 // =0x10
; CHECK-NEXT:    cmp x9, #16
; CHECK-NEXT:    add x10, x10, #16
; CHECK-NEXT:    csel x9, x9, x10, ne
; CHECK-NEXT:    dup v16.2d, x9
; CHECK-NEXT:    adrp x9, .LCPI17_0
; CHECK-NEXT:    cmhi v1.2d, v16.2d, v1.2d
; CHECK-NEXT:    cmhi v2.2d, v16.2d, v2.2d
; CHECK-NEXT:    cmhi v3.2d, v16.2d, v3.2d
; CHECK-NEXT:    cmhi v4.2d, v16.2d, v4.2d
; CHECK-NEXT:    cmhi v5.2d, v16.2d, v5.2d
; CHECK-NEXT:    cmhi v17.2d, v16.2d, v17.2d
; CHECK-NEXT:    cmhi v18.2d, v16.2d, v18.2d
; CHECK-NEXT:    cmhi v19.2d, v16.2d, v19.2d
; CHECK-NEXT:    cmhi v20.2d, v16.2d, v20.2d
; CHECK-NEXT:    cmhi v21.2d, v16.2d, v21.2d
; CHECK-NEXT:    cmhi v22.2d, v16.2d, v22.2d
; CHECK-NEXT:    cmhi v23.2d, v16.2d, v23.2d
; CHECK-NEXT:    cmhi v24.2d, v16.2d, v24.2d
; CHECK-NEXT:    cmhi v6.2d, v16.2d, v6.2d
; CHECK-NEXT:    cmhi v7.2d, v16.2d, v7.2d
; CHECK-NEXT:    cmhi v0.2d, v16.2d, v0.2d
; CHECK-NEXT:    uzp1 v1.4s, v2.4s, v1.4s
; CHECK-NEXT:    uzp1 v2.4s, v18.4s, v17.4s
; CHECK-NEXT:    uzp1 v16.4s, v20.4s, v19.4s
; CHECK-NEXT:    uzp1 v17.4s, v22.4s, v21.4s
; CHECK-NEXT:    uzp1 v3.4s, v4.4s, v3.4s
; CHECK-NEXT:    uzp1 v18.4s, v24.4s, v23.4s
; CHECK-NEXT:    uzp1 v4.4s, v6.4s, v5.4s
; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v7.4s
; CHECK-NEXT:    uzp1 v2.8h, v16.8h, v2.8h
; CHECK-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
; CHECK-NEXT:    uzp1 v5.8h, v18.8h, v17.8h
; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
; CHECK-NEXT:    uzp1 v2.16b, v5.16b, v2.16b
; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    shl v1.16b, v2.16b, #7
; CHECK-NEXT:    ldr q2, [x9, :lo12:.LCPI17_0]
; CHECK-NEXT:    shl v0.16b, v0.16b, #7
; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT:    addp v0.16b, v0.16b, v0.16b
; CHECK-NEXT:    str h1, [x8, #2]
; CHECK-NEXT:    str h0, [x8]
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.v32i1(<32 x i1> %mask.in, i1 false)
  %mask.out = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 %tz.elts)
  ret <32 x i1> %mask.out
}

;; Non-matches
define <vscale x 16 x i1> @mask_exclude_active_nxv16_nonzero_lower_bound(<vscale x 16 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_nxv16_nonzero_lower_bound:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.b
; CHECK-NEXT:    mov w9, #1 // =0x1
; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
; CHECK-NEXT:    cntp x8, p0, p0.b
; CHECK-NEXT:    whilelo p0.b, x9, x8
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> %mask.in, i1 false)
  %mask.out = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 1, i64 %tz.elts)
  ret <vscale x 16 x i1> %mask.out
}

define <vscale x 4 x i1> @mask_exclude_active_narrower_result_type(<vscale x 8 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_narrower_result_type:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.h
; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
; CHECK-NEXT:    cntp x8, p0, p0.h
; CHECK-NEXT:    whilelo p0.s, xzr, x8
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts(<vscale x 8 x i1> %mask.in, i1 false)
  %mask.out = call <vscale x 4 x i1> @llvm.get.active.lane.mask(i64 0, i64 %tz.elts)
  ret <vscale x 4 x i1> %mask.out
}

define <vscale x 16 x i1> @mask_exclude_active_wider_result_type(<vscale x 8 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_wider_result_type:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.h
; CHECK-NEXT:    brkb p0.b, p1/z, p0.b
; CHECK-NEXT:    cntp x8, p0, p0.h
; CHECK-NEXT:    whilelo p0.b, xzr, x8
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts(<vscale x 8 x i1> %mask.in, i1 false)
  %mask.out = call <vscale x 16 x i1> @llvm.get.active.lane.mask(i64 0, i64 %tz.elts)
  ret <vscale x 16 x i1> %mask.out
}

define <4 x i1> @mask_exclude_active_narrower_result_type_fixed(<8 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_narrower_result_type_fixed:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shl v0.8b, v0.8b, #7
; CHECK-NEXT:    ptrue p0.b, vl8
; CHECK-NEXT:    cmpne p1.b, p0/z, z0.b, #0
; CHECK-NEXT:    brkb p1.b, p0/z, p1.b
; CHECK-NEXT:    cntp x8, p1, p1.b
; CHECK-NEXT:    whilelo p0.h, xzr, x8
; CHECK-NEXT:    mov z0.h, p0/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts(<8 x i1> %mask.in, i1 false)
  %mask.out = call <4 x i1> @llvm.get.active.lane.mask(i64 0, i64 %tz.elts)
  ret <4 x i1> %mask.out
}

define <16 x i1> @mask_exclude_active_wider_result_type_fixed(<8 x i1> %mask.in) {
; CHECK-LABEL: mask_exclude_active_wider_result_type_fixed:
; CHECK:       // %bb.0:
; CHECK-NEXT:    shl v0.8b, v0.8b, #7
; CHECK-NEXT:    ptrue p0.b, vl8
; CHECK-NEXT:    cmpne p1.b, p0/z, z0.b, #0
; CHECK-NEXT:    brkb p1.b, p0/z, p1.b
; CHECK-NEXT:    mov z0.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT:    ret
  %tz.elts = call i64 @llvm.experimental.cttz.elts(<8 x i1> %mask.in, i1 false)
  %mask.out = call <16 x i1> @llvm.get.active.lane.mask(i64 0, i64 %tz.elts)
  ret <16 x i1> %mask.out
}