llvm/test/CodeGen/AArch64/default-partial-reduce.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s | FileCheck %s --check-prefixes=NEON-NOBF16
; RUN: llc -mattr=+bf16 < %s | FileCheck %s --check-prefixes=NEON-BF16
; RUN: llc -mattr=+sve2,+bf16,+sve-b16b16, < %s | FileCheck %s --check-prefixes=SVE

target triple = "aarch64-unknown-linux-gnu"

;; Test partial reductions in place of normal vector operations of the same
;; size.

;; Int
define <4 x i8> @pr_4xi8(<4 x i8> %in, <4 x i8> %acc) {
; NEON-NOBF16-LABEL: pr_4xi8:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    bic v0.4h, #255, lsl #8
; NEON-NOBF16-NEXT:    add v0.4h, v1.4h, v0.4h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_4xi8:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    bic v0.4h, #255, lsl #8
; NEON-BF16-NEXT:    add v0.4h, v1.4h, v0.4h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_4xi8:
; SVE:       // %bb.0:
; SVE-NEXT:    bic v0.4h, #255, lsl #8
; SVE-NEXT:    add v0.4h, v1.4h, v0.4h
; SVE-NEXT:    ret
  %res = call <4 x i8> @llvm.vector.partial.reduce.add(<4 x i8> %acc, <4 x i8> %in)
  ret <4 x i8> %res
}

define <8 x i8> @pr_8xi8(<8 x i8> %in, <8 x i8> %acc) {
; NEON-NOBF16-LABEL: pr_8xi8:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add v0.8b, v1.8b, v0.8b
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_8xi8:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add v0.8b, v1.8b, v0.8b
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_8xi8:
; SVE:       // %bb.0:
; SVE-NEXT:    add v0.8b, v1.8b, v0.8b
; SVE-NEXT:    ret
  %res = call <8 x i8> @llvm.vector.partial.reduce.add(<8 x i8> %acc, <8 x i8> %in)
  ret <8 x i8> %res
}

define <16 x i8> @pr_16xi8(<16 x i8> %in, <16 x i8> %acc) {
; NEON-NOBF16-LABEL: pr_16xi8:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add v0.16b, v1.16b, v0.16b
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_16xi8:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add v0.16b, v1.16b, v0.16b
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_16xi8:
; SVE:       // %bb.0:
; SVE-NEXT:    add v0.16b, v1.16b, v0.16b
; SVE-NEXT:    ret
  %res = call <16 x i8> @llvm.vector.partial.reduce.add(<16 x i8> %acc, <16 x i8> %in)
  ret <16 x i8> %res
}

define <32 x i8> @pr_32xi8(<32 x i8> %in, <32 x i8> %acc) {
; NEON-NOBF16-LABEL: pr_32xi8:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add v1.16b, v3.16b, v1.16b
; NEON-NOBF16-NEXT:    add v0.16b, v2.16b, v0.16b
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_32xi8:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add v1.16b, v3.16b, v1.16b
; NEON-BF16-NEXT:    add v0.16b, v2.16b, v0.16b
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_32xi8:
; SVE:       // %bb.0:
; SVE-NEXT:    add v1.16b, v3.16b, v1.16b
; SVE-NEXT:    add v0.16b, v2.16b, v0.16b
; SVE-NEXT:    ret
  %res = call <32 x i8> @llvm.vector.partial.reduce.add(<32 x i8> %acc, <32 x i8> %in)
  ret <32 x i8> %res
}

define <2 x i16> @pr_2xi16(<2 x i16> %in, <2 x i16> %acc) {
; NEON-NOBF16-LABEL: pr_2xi16:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    movi d2, #0x00ffff0000ffff
; NEON-NOBF16-NEXT:    and v0.8b, v0.8b, v2.8b
; NEON-NOBF16-NEXT:    add v0.2s, v1.2s, v0.2s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_2xi16:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    movi d2, #0x00ffff0000ffff
; NEON-BF16-NEXT:    and v0.8b, v0.8b, v2.8b
; NEON-BF16-NEXT:    add v0.2s, v1.2s, v0.2s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_2xi16:
; SVE:       // %bb.0:
; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
; SVE-NEXT:    and z0.s, z0.s, #0xffff
; SVE-NEXT:    add v0.2s, v1.2s, v0.2s
; SVE-NEXT:    ret
  %res = call <2 x i16> @llvm.vector.partial.reduce.add(<2 x i16> %acc, <2 x i16> %in)
  ret <2 x i16> %res
}

define <4 x i16> @pr_4xi16(<4 x i16> %in, <4 x i16> %acc) {
; NEON-NOBF16-LABEL: pr_4xi16:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add v0.4h, v1.4h, v0.4h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_4xi16:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add v0.4h, v1.4h, v0.4h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_4xi16:
; SVE:       // %bb.0:
; SVE-NEXT:    add v0.4h, v1.4h, v0.4h
; SVE-NEXT:    ret
  %res = call <4 x i16> @llvm.vector.partial.reduce.add(<4 x i16> %acc, <4 x i16> %in)
  ret <4 x i16> %res
}

define <8 x i16> @pr_8xi16(<8 x i16> %in, <8 x i16> %acc) {
; NEON-NOBF16-LABEL: pr_8xi16:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add v0.8h, v1.8h, v0.8h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_8xi16:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add v0.8h, v1.8h, v0.8h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_8xi16:
; SVE:       // %bb.0:
; SVE-NEXT:    add v0.8h, v1.8h, v0.8h
; SVE-NEXT:    ret
  %res = call <8 x i16> @llvm.vector.partial.reduce.add(<8 x i16> %acc, <8 x i16> %in)
  ret <8 x i16> %res
}

define <16 x i16> @pr_16xi16(<16 x i16> %in, <16 x i16> %acc) {
; NEON-NOBF16-LABEL: pr_16xi16:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add v1.8h, v3.8h, v1.8h
; NEON-NOBF16-NEXT:    add v0.8h, v2.8h, v0.8h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_16xi16:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add v1.8h, v3.8h, v1.8h
; NEON-BF16-NEXT:    add v0.8h, v2.8h, v0.8h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_16xi16:
; SVE:       // %bb.0:
; SVE-NEXT:    add v1.8h, v3.8h, v1.8h
; SVE-NEXT:    add v0.8h, v2.8h, v0.8h
; SVE-NEXT:    ret
  %res = call <16 x i16> @llvm.vector.partial.reduce.add(<16 x i16> %acc, <16 x i16> %in)
  ret <16 x i16> %res
}

define <2 x i32> @pr_2xi32(<2 x i32> %in, <2 x i32> %acc) {
; NEON-NOBF16-LABEL: pr_2xi32:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add v0.2s, v1.2s, v0.2s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_2xi32:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add v0.2s, v1.2s, v0.2s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_2xi32:
; SVE:       // %bb.0:
; SVE-NEXT:    add v0.2s, v1.2s, v0.2s
; SVE-NEXT:    ret
  %res = call <2 x i32> @llvm.vector.partial.reduce.add(<2 x i32> %acc, <2 x i32> %in)
  ret <2 x i32> %res
}

define <4 x i32> @pr_4xi32(<4 x i32> %in, <4 x i32> %acc) {
; NEON-NOBF16-LABEL: pr_4xi32:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add v0.4s, v1.4s, v0.4s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_4xi32:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add v0.4s, v1.4s, v0.4s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_4xi32:
; SVE:       // %bb.0:
; SVE-NEXT:    add v0.4s, v1.4s, v0.4s
; SVE-NEXT:    ret
  %res = call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> %acc, <4 x i32> %in)
  ret <4 x i32> %res
}

define <8 x i32> @pr_8xi32(<8 x i32> %in, <8 x i32> %acc) {
; NEON-NOBF16-LABEL: pr_8xi32:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add v1.4s, v3.4s, v1.4s
; NEON-NOBF16-NEXT:    add v0.4s, v2.4s, v0.4s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_8xi32:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add v1.4s, v3.4s, v1.4s
; NEON-BF16-NEXT:    add v0.4s, v2.4s, v0.4s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_8xi32:
; SVE:       // %bb.0:
; SVE-NEXT:    add v1.4s, v3.4s, v1.4s
; SVE-NEXT:    add v0.4s, v2.4s, v0.4s
; SVE-NEXT:    ret
  %res = call <8 x i32> @llvm.vector.partial.reduce.add(<8 x i32> %acc, <8 x i32> %in)
  ret <8 x i32> %res
}

define <2 x i64> @pr_2xi64(<2 x i64> %in, <2 x i64> %acc) {
; NEON-NOBF16-LABEL: pr_2xi64:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add v0.2d, v1.2d, v0.2d
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_2xi64:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add v0.2d, v1.2d, v0.2d
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_2xi64:
; SVE:       // %bb.0:
; SVE-NEXT:    add v0.2d, v1.2d, v0.2d
; SVE-NEXT:    ret
  %res = call <2 x i64> @llvm.vector.partial.reduce.add(<2 x i64> %acc, <2 x i64> %in)
  ret <2 x i64> %res
}

define <4 x i64> @pr_4xi64(<4 x i64> %in, <4 x i64> %acc) {
; NEON-NOBF16-LABEL: pr_4xi64:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add v1.2d, v3.2d, v1.2d
; NEON-NOBF16-NEXT:    add v0.2d, v2.2d, v0.2d
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_4xi64:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add v1.2d, v3.2d, v1.2d
; NEON-BF16-NEXT:    add v0.2d, v2.2d, v0.2d
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_4xi64:
; SVE:       // %bb.0:
; SVE-NEXT:    add v1.2d, v3.2d, v1.2d
; SVE-NEXT:    add v0.2d, v2.2d, v0.2d
; SVE-NEXT:    ret
  %res = call <4 x i64> @llvm.vector.partial.reduce.add(<4 x i64> %acc, <4 x i64> %in)
  ret <4 x i64> %res
}

define <vscale x 8 x i8> @pr_nx8xi8(<vscale x 8 x i8> %in, <vscale x 8 x i8> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx8xi8:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    uaddwb z0.h, z1.h, z0.b
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx8xi8:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    uaddwb z0.h, z1.h, z0.b
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx8xi8:
; SVE:       // %bb.0:
; SVE-NEXT:    uaddwb z0.h, z1.h, z0.b
; SVE-NEXT:    ret
  %res = call <vscale x 8 x i8> @llvm.vector.partial.reduce.add(<vscale x 8 x i8> %acc, <vscale x 8 x i8> %in)
  ret <vscale x 8 x i8> %res
}

define <vscale x 16 x i8> @pr_nx16xi8(<vscale x 16 x i8> %in, <vscale x 16 x i8> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx16xi8:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add z0.b, z1.b, z0.b
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx16xi8:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add z0.b, z1.b, z0.b
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx16xi8:
; SVE:       // %bb.0:
; SVE-NEXT:    add z0.b, z1.b, z0.b
; SVE-NEXT:    ret
  %res = call <vscale x 16 x i8> @llvm.vector.partial.reduce.add(<vscale x 16 x i8> %acc, <vscale x 16 x i8> %in)
  ret <vscale x 16 x i8> %res
}

define <vscale x 32 x i8> @pr_nx32xi8(<vscale x 32 x i8> %in, <vscale x 32 x i8> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx32xi8:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add z1.b, z3.b, z1.b
; NEON-NOBF16-NEXT:    add z0.b, z2.b, z0.b
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx32xi8:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add z1.b, z3.b, z1.b
; NEON-BF16-NEXT:    add z0.b, z2.b, z0.b
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx32xi8:
; SVE:       // %bb.0:
; SVE-NEXT:    add z1.b, z3.b, z1.b
; SVE-NEXT:    add z0.b, z2.b, z0.b
; SVE-NEXT:    ret
  %res = call <vscale x 32 x i8> @llvm.vector.partial.reduce.add(<vscale x 32 x i8> %acc, <vscale x 32 x i8> %in)
  ret <vscale x 32 x i8> %res
}

define <vscale x 4 x i16> @pr_nx4xi16(<vscale x 4 x i16> %in, <vscale x 4 x i16> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx4xi16:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    uaddwb z0.s, z1.s, z0.h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx4xi16:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    uaddwb z0.s, z1.s, z0.h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx4xi16:
; SVE:       // %bb.0:
; SVE-NEXT:    uaddwb z0.s, z1.s, z0.h
; SVE-NEXT:    ret
  %res = call <vscale x 4 x i16> @llvm.vector.partial.reduce.add(<vscale x 4 x i16> %acc, <vscale x 4 x i16> %in)
  ret <vscale x 4 x i16> %res
}

define <vscale x 8 x i16> @pr_nx8xi16(<vscale x 8 x i16> %in, <vscale x 8 x i16> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx8xi16:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add z0.h, z1.h, z0.h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx8xi16:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add z0.h, z1.h, z0.h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx8xi16:
; SVE:       // %bb.0:
; SVE-NEXT:    add z0.h, z1.h, z0.h
; SVE-NEXT:    ret
  %res = call <vscale x 8 x i16> @llvm.vector.partial.reduce.add(<vscale x 8 x i16> %acc, <vscale x 8 x i16> %in)
  ret <vscale x 8 x i16> %res
}

define <vscale x 16 x i16> @pr_nx16xi16(<vscale x 16 x i16> %in, <vscale x 16 x i16> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx16xi16:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add z1.h, z3.h, z1.h
; NEON-NOBF16-NEXT:    add z0.h, z2.h, z0.h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx16xi16:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add z1.h, z3.h, z1.h
; NEON-BF16-NEXT:    add z0.h, z2.h, z0.h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx16xi16:
; SVE:       // %bb.0:
; SVE-NEXT:    add z1.h, z3.h, z1.h
; SVE-NEXT:    add z0.h, z2.h, z0.h
; SVE-NEXT:    ret
  %res = call <vscale x 16 x i16> @llvm.vector.partial.reduce.add(<vscale x 16 x i16> %acc, <vscale x 16 x i16> %in)
  ret <vscale x 16 x i16> %res
}

define <vscale x 2 x i32> @pr_nx2xi32(<vscale x 2 x i32> %in, <vscale x 2 x i32> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx2xi32:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    adr z0.d, [z1.d, z0.d, uxtw]
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx2xi32:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    adr z0.d, [z1.d, z0.d, uxtw]
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx2xi32:
; SVE:       // %bb.0:
; SVE-NEXT:    adr z0.d, [z1.d, z0.d, uxtw]
; SVE-NEXT:    ret
  %res = call <vscale x 2 x i32> @llvm.vector.partial.reduce.add(<vscale x 2 x i32> %acc, <vscale x 2 x i32> %in)
  ret <vscale x 2 x i32> %res
}

define <vscale x 4 x i32> @pr_nx4xi32(<vscale x 4 x i32> %in, <vscale x 4 x i32> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx4xi32:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add z0.s, z1.s, z0.s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx4xi32:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add z0.s, z1.s, z0.s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx4xi32:
; SVE:       // %bb.0:
; SVE-NEXT:    add z0.s, z1.s, z0.s
; SVE-NEXT:    ret
  %res = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add(<vscale x 4 x i32> %acc, <vscale x 4 x i32> %in)
  ret <vscale x 4 x i32> %res
}

define <vscale x 8 x i32> @pr_nx8xi32(<vscale x 8 x i32> %in, <vscale x 8 x i32> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx8xi32:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add z1.s, z3.s, z1.s
; NEON-NOBF16-NEXT:    add z0.s, z2.s, z0.s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx8xi32:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add z1.s, z3.s, z1.s
; NEON-BF16-NEXT:    add z0.s, z2.s, z0.s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx8xi32:
; SVE:       // %bb.0:
; SVE-NEXT:    add z1.s, z3.s, z1.s
; SVE-NEXT:    add z0.s, z2.s, z0.s
; SVE-NEXT:    ret
  %res = call <vscale x 8 x i32> @llvm.vector.partial.reduce.add(<vscale x 8 x i32> %acc, <vscale x 8 x i32> %in)
  ret <vscale x 8 x i32> %res
}

define <vscale x 2 x i64> @pr_nx2xi64(<vscale x 2 x i64> %in, <vscale x 2 x i64> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx2xi64:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add z0.d, z1.d, z0.d
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx2xi64:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add z0.d, z1.d, z0.d
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx2xi64:
; SVE:       // %bb.0:
; SVE-NEXT:    add z0.d, z1.d, z0.d
; SVE-NEXT:    ret
  %res = call <vscale x 2 x i64> @llvm.vector.partial.reduce.add(<vscale x 2 x i64> %acc, <vscale x 2 x i64> %in)
  ret <vscale x 2 x i64> %res
}

define <vscale x 4 x i64> @pr_nx4xi64(<vscale x 4 x i64> %in, <vscale x 4 x i64> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx4xi64:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    add z1.d, z3.d, z1.d
; NEON-NOBF16-NEXT:    add z0.d, z2.d, z0.d
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx4xi64:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    add z1.d, z3.d, z1.d
; NEON-BF16-NEXT:    add z0.d, z2.d, z0.d
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx4xi64:
; SVE:       // %bb.0:
; SVE-NEXT:    add z1.d, z3.d, z1.d
; SVE-NEXT:    add z0.d, z2.d, z0.d
; SVE-NEXT:    ret
  %res = call <vscale x 4 x i64> @llvm.vector.partial.reduce.add(<vscale x 4 x i64> %acc, <vscale x 4 x i64> %in)
  ret <vscale x 4 x i64> %res
}

;; FP
define <2 x half> @pr_2xhalf(<2 x half> %in, <2 x half> %acc) {
; NEON-NOBF16-LABEL: pr_2xhalf:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fcvtl v0.4s, v0.4h
; NEON-NOBF16-NEXT:    fcvtl v1.4s, v1.4h
; NEON-NOBF16-NEXT:    fadd v0.4s, v1.4s, v0.4s
; NEON-NOBF16-NEXT:    fcvtn v0.4h, v0.4s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_2xhalf:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fcvtl v0.4s, v0.4h
; NEON-BF16-NEXT:    fcvtl v1.4s, v1.4h
; NEON-BF16-NEXT:    fadd v0.4s, v1.4s, v0.4s
; NEON-BF16-NEXT:    fcvtn v0.4h, v0.4s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_2xhalf:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd v0.4h, v1.4h, v0.4h
; SVE-NEXT:    ret
  %res = call <2 x half> @llvm.vector.partial.reduce.fadd(<2 x half> %acc, <2 x half> %in)
  ret <2 x half> %res
}

define <4 x half> @pr_4xhalf(<4 x half> %in, <4 x half> %acc) {
; NEON-NOBF16-LABEL: pr_4xhalf:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fcvtl v0.4s, v0.4h
; NEON-NOBF16-NEXT:    fcvtl v1.4s, v1.4h
; NEON-NOBF16-NEXT:    fadd v0.4s, v1.4s, v0.4s
; NEON-NOBF16-NEXT:    fcvtn v0.4h, v0.4s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_4xhalf:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fcvtl v0.4s, v0.4h
; NEON-BF16-NEXT:    fcvtl v1.4s, v1.4h
; NEON-BF16-NEXT:    fadd v0.4s, v1.4s, v0.4s
; NEON-BF16-NEXT:    fcvtn v0.4h, v0.4s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_4xhalf:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd v0.4h, v1.4h, v0.4h
; SVE-NEXT:    ret
  %res = call <4 x half> @llvm.vector.partial.reduce.fadd(<4 x half> %acc, <4 x half> %in)
  ret <4 x half> %res
}

define <8 x half> @pr_8xhalf(<8 x half> %in, <8 x half> %acc) {
; NEON-NOBF16-LABEL: pr_8xhalf:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fcvtl v2.4s, v0.4h
; NEON-NOBF16-NEXT:    fcvtl v3.4s, v1.4h
; NEON-NOBF16-NEXT:    fcvtl2 v0.4s, v0.8h
; NEON-NOBF16-NEXT:    fcvtl2 v1.4s, v1.8h
; NEON-NOBF16-NEXT:    fadd v2.4s, v3.4s, v2.4s
; NEON-NOBF16-NEXT:    fadd v1.4s, v1.4s, v0.4s
; NEON-NOBF16-NEXT:    fcvtn v0.4h, v2.4s
; NEON-NOBF16-NEXT:    fcvtn2 v0.8h, v1.4s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_8xhalf:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fcvtl v2.4s, v0.4h
; NEON-BF16-NEXT:    fcvtl v3.4s, v1.4h
; NEON-BF16-NEXT:    fcvtl2 v0.4s, v0.8h
; NEON-BF16-NEXT:    fcvtl2 v1.4s, v1.8h
; NEON-BF16-NEXT:    fadd v2.4s, v3.4s, v2.4s
; NEON-BF16-NEXT:    fadd v1.4s, v1.4s, v0.4s
; NEON-BF16-NEXT:    fcvtn v0.4h, v2.4s
; NEON-BF16-NEXT:    fcvtn2 v0.8h, v1.4s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_8xhalf:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd v0.8h, v1.8h, v0.8h
; SVE-NEXT:    ret
  %res = call <8 x half> @llvm.vector.partial.reduce.fadd(<8 x half> %acc, <8 x half> %in)
  ret <8 x half> %res
}

define <16 x half> @pr_16xhalf(<16 x half> %in, <16 x half> %acc) {
; NEON-NOBF16-LABEL: pr_16xhalf:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fcvtl v4.4s, v2.4h
; NEON-NOBF16-NEXT:    fcvtl v5.4s, v0.4h
; NEON-NOBF16-NEXT:    fcvtl v6.4s, v1.4h
; NEON-NOBF16-NEXT:    fcvtl v7.4s, v3.4h
; NEON-NOBF16-NEXT:    fcvtl2 v0.4s, v0.8h
; NEON-NOBF16-NEXT:    fcvtl2 v2.4s, v2.8h
; NEON-NOBF16-NEXT:    fcvtl2 v1.4s, v1.8h
; NEON-NOBF16-NEXT:    fcvtl2 v3.4s, v3.8h
; NEON-NOBF16-NEXT:    fadd v4.4s, v4.4s, v5.4s
; NEON-NOBF16-NEXT:    fadd v5.4s, v7.4s, v6.4s
; NEON-NOBF16-NEXT:    fadd v2.4s, v2.4s, v0.4s
; NEON-NOBF16-NEXT:    fadd v3.4s, v3.4s, v1.4s
; NEON-NOBF16-NEXT:    fcvtn v0.4h, v4.4s
; NEON-NOBF16-NEXT:    fcvtn v1.4h, v5.4s
; NEON-NOBF16-NEXT:    fcvtn2 v0.8h, v2.4s
; NEON-NOBF16-NEXT:    fcvtn2 v1.8h, v3.4s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_16xhalf:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fcvtl v4.4s, v2.4h
; NEON-BF16-NEXT:    fcvtl v5.4s, v0.4h
; NEON-BF16-NEXT:    fcvtl v6.4s, v1.4h
; NEON-BF16-NEXT:    fcvtl v7.4s, v3.4h
; NEON-BF16-NEXT:    fcvtl2 v0.4s, v0.8h
; NEON-BF16-NEXT:    fcvtl2 v2.4s, v2.8h
; NEON-BF16-NEXT:    fcvtl2 v1.4s, v1.8h
; NEON-BF16-NEXT:    fcvtl2 v3.4s, v3.8h
; NEON-BF16-NEXT:    fadd v4.4s, v4.4s, v5.4s
; NEON-BF16-NEXT:    fadd v5.4s, v7.4s, v6.4s
; NEON-BF16-NEXT:    fadd v2.4s, v2.4s, v0.4s
; NEON-BF16-NEXT:    fadd v3.4s, v3.4s, v1.4s
; NEON-BF16-NEXT:    fcvtn v0.4h, v4.4s
; NEON-BF16-NEXT:    fcvtn v1.4h, v5.4s
; NEON-BF16-NEXT:    fcvtn2 v0.8h, v2.4s
; NEON-BF16-NEXT:    fcvtn2 v1.8h, v3.4s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_16xhalf:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd v1.8h, v3.8h, v1.8h
; SVE-NEXT:    fadd v0.8h, v2.8h, v0.8h
; SVE-NEXT:    ret
  %res = call <16 x half> @llvm.vector.partial.reduce.fadd(<16 x half> %acc, <16 x half> %in)
  ret <16 x half> %res
}

define <2 x bfloat> @pr_2xbfloat(<2 x bfloat> %in, <2 x bfloat> %acc) {
; NEON-NOBF16-LABEL: pr_2xbfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    shll v0.4s, v0.4h, #16
; NEON-NOBF16-NEXT:    shll v1.4s, v1.4h, #16
; NEON-NOBF16-NEXT:    fadd v0.4s, v1.4s, v0.4s
; NEON-NOBF16-NEXT:    movi v1.4s, #1
; NEON-NOBF16-NEXT:    ushr v2.4s, v0.4s, #16
; NEON-NOBF16-NEXT:    and v1.16b, v2.16b, v1.16b
; NEON-NOBF16-NEXT:    movi v2.4s, #127, msl #8
; NEON-NOBF16-NEXT:    add v0.4s, v1.4s, v0.4s
; NEON-NOBF16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_2xbfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    shll v0.4s, v0.4h, #16
; NEON-BF16-NEXT:    shll v1.4s, v1.4h, #16
; NEON-BF16-NEXT:    fadd v0.4s, v1.4s, v0.4s
; NEON-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_2xbfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.h, vl4
; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
; SVE-NEXT:    // kill: def $d1 killed $d1 def $z1
; SVE-NEXT:    bfadd z0.h, p0/m, z0.h, z1.h
; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT:    ret
  %res = call <2 x bfloat> @llvm.vector.partial.reduce.fadd(<2 x bfloat> %acc, <2 x bfloat> %in)
  ret <2 x bfloat> %res
}

define <4 x bfloat> @pr_4xbfloat(<4 x bfloat> %in, <4 x bfloat> %acc) {
; NEON-NOBF16-LABEL: pr_4xbfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    shll v0.4s, v0.4h, #16
; NEON-NOBF16-NEXT:    shll v1.4s, v1.4h, #16
; NEON-NOBF16-NEXT:    fadd v0.4s, v1.4s, v0.4s
; NEON-NOBF16-NEXT:    movi v1.4s, #1
; NEON-NOBF16-NEXT:    ushr v2.4s, v0.4s, #16
; NEON-NOBF16-NEXT:    and v1.16b, v2.16b, v1.16b
; NEON-NOBF16-NEXT:    movi v2.4s, #127, msl #8
; NEON-NOBF16-NEXT:    add v0.4s, v1.4s, v0.4s
; NEON-NOBF16-NEXT:    addhn v0.4h, v0.4s, v2.4s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_4xbfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    shll v0.4s, v0.4h, #16
; NEON-BF16-NEXT:    shll v1.4s, v1.4h, #16
; NEON-BF16-NEXT:    fadd v0.4s, v1.4s, v0.4s
; NEON-BF16-NEXT:    bfcvtn v0.4h, v0.4s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_4xbfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.h, vl4
; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
; SVE-NEXT:    // kill: def $d1 killed $d1 def $z1
; SVE-NEXT:    bfadd z0.h, p0/m, z0.h, z1.h
; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
; SVE-NEXT:    ret
  %res = call <4 x bfloat> @llvm.vector.partial.reduce.fadd(<4 x bfloat> %acc, <4 x bfloat> %in)
  ret <4 x bfloat> %res
}

define <8 x bfloat> @pr_8xbfloat(<8 x bfloat> %in, <8 x bfloat> %acc) {
; NEON-NOBF16-LABEL: pr_8xbfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    shll v3.4s, v0.4h, #16
; NEON-NOBF16-NEXT:    shll v4.4s, v1.4h, #16
; NEON-NOBF16-NEXT:    shll2 v0.4s, v0.8h, #16
; NEON-NOBF16-NEXT:    shll2 v1.4s, v1.8h, #16
; NEON-NOBF16-NEXT:    movi v2.4s, #1
; NEON-NOBF16-NEXT:    movi v5.4s, #127, msl #8
; NEON-NOBF16-NEXT:    fadd v3.4s, v4.4s, v3.4s
; NEON-NOBF16-NEXT:    fadd v1.4s, v1.4s, v0.4s
; NEON-NOBF16-NEXT:    ushr v0.4s, v3.4s, #16
; NEON-NOBF16-NEXT:    ushr v4.4s, v1.4s, #16
; NEON-NOBF16-NEXT:    and v0.16b, v0.16b, v2.16b
; NEON-NOBF16-NEXT:    and v2.16b, v4.16b, v2.16b
; NEON-NOBF16-NEXT:    add v0.4s, v0.4s, v3.4s
; NEON-NOBF16-NEXT:    add v1.4s, v2.4s, v1.4s
; NEON-NOBF16-NEXT:    addhn v0.4h, v0.4s, v5.4s
; NEON-NOBF16-NEXT:    addhn2 v0.8h, v1.4s, v5.4s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_8xbfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    shll v2.4s, v0.4h, #16
; NEON-BF16-NEXT:    shll v3.4s, v1.4h, #16
; NEON-BF16-NEXT:    shll2 v0.4s, v0.8h, #16
; NEON-BF16-NEXT:    shll2 v1.4s, v1.8h, #16
; NEON-BF16-NEXT:    fadd v2.4s, v3.4s, v2.4s
; NEON-BF16-NEXT:    fadd v1.4s, v1.4s, v0.4s
; NEON-BF16-NEXT:    bfcvtn v0.4h, v2.4s
; NEON-BF16-NEXT:    bfcvtn2 v0.8h, v1.4s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_8xbfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.h, vl8
; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
; SVE-NEXT:    // kill: def $q1 killed $q1 def $z1
; SVE-NEXT:    bfadd z0.h, p0/m, z0.h, z1.h
; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT:    ret
  %res = call <8 x bfloat> @llvm.vector.partial.reduce.fadd(<8 x bfloat> %acc, <8 x bfloat> %in)
  ret <8 x bfloat> %res
}

define <16 x bfloat> @pr_16xbfloat(<16 x bfloat> %in, <16 x bfloat> %acc) {
; NEON-NOBF16-LABEL: pr_16xbfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    shll v4.4s, v2.4h, #16
; NEON-NOBF16-NEXT:    shll v5.4s, v0.4h, #16
; NEON-NOBF16-NEXT:    shll v6.4s, v1.4h, #16
; NEON-NOBF16-NEXT:    shll v7.4s, v3.4h, #16
; NEON-NOBF16-NEXT:    shll2 v0.4s, v0.8h, #16
; NEON-NOBF16-NEXT:    shll2 v2.4s, v2.8h, #16
; NEON-NOBF16-NEXT:    shll2 v1.4s, v1.8h, #16
; NEON-NOBF16-NEXT:    shll2 v3.4s, v3.8h, #16
; NEON-NOBF16-NEXT:    movi v17.4s, #127, msl #8
; NEON-NOBF16-NEXT:    fadd v4.4s, v4.4s, v5.4s
; NEON-NOBF16-NEXT:    fadd v5.4s, v7.4s, v6.4s
; NEON-NOBF16-NEXT:    fadd v2.4s, v2.4s, v0.4s
; NEON-NOBF16-NEXT:    movi v0.4s, #1
; NEON-NOBF16-NEXT:    fadd v3.4s, v3.4s, v1.4s
; NEON-NOBF16-NEXT:    ushr v6.4s, v4.4s, #16
; NEON-NOBF16-NEXT:    ushr v1.4s, v5.4s, #16
; NEON-NOBF16-NEXT:    ushr v7.4s, v2.4s, #16
; NEON-NOBF16-NEXT:    ushr v16.4s, v3.4s, #16
; NEON-NOBF16-NEXT:    and v6.16b, v6.16b, v0.16b
; NEON-NOBF16-NEXT:    and v1.16b, v1.16b, v0.16b
; NEON-NOBF16-NEXT:    add v4.4s, v6.4s, v4.4s
; NEON-NOBF16-NEXT:    and v6.16b, v16.16b, v0.16b
; NEON-NOBF16-NEXT:    add v1.4s, v1.4s, v5.4s
; NEON-NOBF16-NEXT:    and v5.16b, v7.16b, v0.16b
; NEON-NOBF16-NEXT:    addhn v0.4h, v4.4s, v17.4s
; NEON-NOBF16-NEXT:    add v3.4s, v6.4s, v3.4s
; NEON-NOBF16-NEXT:    addhn v1.4h, v1.4s, v17.4s
; NEON-NOBF16-NEXT:    add v2.4s, v5.4s, v2.4s
; NEON-NOBF16-NEXT:    addhn2 v0.8h, v2.4s, v17.4s
; NEON-NOBF16-NEXT:    addhn2 v1.8h, v3.4s, v17.4s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_16xbfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    shll v4.4s, v2.4h, #16
; NEON-BF16-NEXT:    shll v5.4s, v0.4h, #16
; NEON-BF16-NEXT:    shll v6.4s, v1.4h, #16
; NEON-BF16-NEXT:    shll v7.4s, v3.4h, #16
; NEON-BF16-NEXT:    shll2 v0.4s, v0.8h, #16
; NEON-BF16-NEXT:    shll2 v2.4s, v2.8h, #16
; NEON-BF16-NEXT:    shll2 v1.4s, v1.8h, #16
; NEON-BF16-NEXT:    shll2 v3.4s, v3.8h, #16
; NEON-BF16-NEXT:    fadd v4.4s, v4.4s, v5.4s
; NEON-BF16-NEXT:    fadd v5.4s, v7.4s, v6.4s
; NEON-BF16-NEXT:    fadd v2.4s, v2.4s, v0.4s
; NEON-BF16-NEXT:    fadd v3.4s, v3.4s, v1.4s
; NEON-BF16-NEXT:    bfcvtn v0.4h, v4.4s
; NEON-BF16-NEXT:    bfcvtn v1.4h, v5.4s
; NEON-BF16-NEXT:    bfcvtn2 v0.8h, v2.4s
; NEON-BF16-NEXT:    bfcvtn2 v1.8h, v3.4s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_16xbfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.h, vl8
; SVE-NEXT:    // kill: def $q1 killed $q1 def $z1
; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
; SVE-NEXT:    // kill: def $q3 killed $q3 def $z3
; SVE-NEXT:    // kill: def $q2 killed $q2 def $z2
; SVE-NEXT:    bfadd z0.h, p0/m, z0.h, z2.h
; SVE-NEXT:    bfadd z1.h, p0/m, z1.h, z3.h
; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT:    // kill: def $q1 killed $q1 killed $z1
; SVE-NEXT:    ret
  %res = call <16 x bfloat> @llvm.vector.partial.reduce.fadd(<16 x bfloat> %acc, <16 x bfloat> %in)
  ret <16 x bfloat> %res
}

define <2 x float> @pr_2xfloat(<2 x float> %in, <2 x float> %acc) {
; NEON-NOBF16-LABEL: pr_2xfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fadd v0.2s, v1.2s, v0.2s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_2xfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fadd v0.2s, v1.2s, v0.2s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_2xfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd v0.2s, v1.2s, v0.2s
; SVE-NEXT:    ret
  %res = call <2 x float> @llvm.vector.partial.reduce.fadd(<2 x float> %acc, <2 x float> %in)
  ret <2 x float> %res
}

define <4 x float> @pr_4xfloat(<4 x float> %in, <4 x float> %acc) {
; NEON-NOBF16-LABEL: pr_4xfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fadd v0.4s, v1.4s, v0.4s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_4xfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fadd v0.4s, v1.4s, v0.4s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_4xfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd v0.4s, v1.4s, v0.4s
; SVE-NEXT:    ret
  %res = call <4 x float> @llvm.vector.partial.reduce.fadd(<4 x float> %acc, <4 x float> %in)
  ret <4 x float> %res
}

define <8 x float> @pr_8xfloat(<8 x float> %in, <8 x float> %acc) {
; NEON-NOBF16-LABEL: pr_8xfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fadd v1.4s, v3.4s, v1.4s
; NEON-NOBF16-NEXT:    fadd v0.4s, v2.4s, v0.4s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_8xfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fadd v1.4s, v3.4s, v1.4s
; NEON-BF16-NEXT:    fadd v0.4s, v2.4s, v0.4s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_8xfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd v1.4s, v3.4s, v1.4s
; SVE-NEXT:    fadd v0.4s, v2.4s, v0.4s
; SVE-NEXT:    ret
  %res = call <8 x float> @llvm.vector.partial.reduce.fadd(<8 x float> %acc, <8 x float> %in)
  ret <8 x float> %res
}

define <2 x double> @pr_2xdouble(<2 x double> %in, <2 x double> %acc) {
; NEON-NOBF16-LABEL: pr_2xdouble:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fadd v0.2d, v1.2d, v0.2d
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_2xdouble:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fadd v0.2d, v1.2d, v0.2d
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_2xdouble:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd v0.2d, v1.2d, v0.2d
; SVE-NEXT:    ret
  %res = call <2 x double> @llvm.vector.partial.reduce.fadd(<2 x double> %acc, <2 x double> %in)
  ret <2 x double> %res
}

define <4 x double> @pr_4xdouble(<4 x double> %in, <4 x double> %acc) {
; NEON-NOBF16-LABEL: pr_4xdouble:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fadd v1.2d, v3.2d, v1.2d
; NEON-NOBF16-NEXT:    fadd v0.2d, v2.2d, v0.2d
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_4xdouble:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fadd v1.2d, v3.2d, v1.2d
; NEON-BF16-NEXT:    fadd v0.2d, v2.2d, v0.2d
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_4xdouble:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd v1.2d, v3.2d, v1.2d
; SVE-NEXT:    fadd v0.2d, v2.2d, v0.2d
; SVE-NEXT:    ret
  %res = call <4 x double> @llvm.vector.partial.reduce.fadd(<4 x double> %acc, <4 x double> %in)
  ret <4 x double> %res
}

define <vscale x 4 x half> @pr_nx4xhalf(<vscale x 4 x half> %in, <vscale x 4 x half> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx4xhalf:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    ptrue p0.s
; NEON-NOBF16-NEXT:    fadd z0.h, p0/m, z0.h, z1.h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx4xhalf:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    ptrue p0.s
; NEON-BF16-NEXT:    fadd z0.h, p0/m, z0.h, z1.h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx4xhalf:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.s
; SVE-NEXT:    fadd z0.h, p0/m, z0.h, z1.h
; SVE-NEXT:    ret
  %res = call <vscale x 4 x half> @llvm.vector.partial.reduce.fadd(<vscale x 4 x half> %acc, <vscale x 4 x half> %in)
  ret <vscale x 4 x half> %res
}

define <vscale x 8 x half> @pr_nx8xhalf(<vscale x 8 x half> %in, <vscale x 8 x half> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx8xhalf:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fadd z0.h, z1.h, z0.h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx8xhalf:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fadd z0.h, z1.h, z0.h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx8xhalf:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd z0.h, z1.h, z0.h
; SVE-NEXT:    ret
  %res = call <vscale x 8 x half> @llvm.vector.partial.reduce.fadd(<vscale x 8 x half> %acc, <vscale x 8 x half> %in)
  ret <vscale x 8 x half> %res
}

define <vscale x 16 x half> @pr_nx16xhalf(<vscale x 16 x half> %in, <vscale x 16 x half> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx16xhalf:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fadd z1.h, z3.h, z1.h
; NEON-NOBF16-NEXT:    fadd z0.h, z2.h, z0.h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx16xhalf:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fadd z1.h, z3.h, z1.h
; NEON-BF16-NEXT:    fadd z0.h, z2.h, z0.h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx16xhalf:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd z1.h, z3.h, z1.h
; SVE-NEXT:    fadd z0.h, z2.h, z0.h
; SVE-NEXT:    ret
  %res = call <vscale x 16 x half> @llvm.vector.partial.reduce.fadd(<vscale x 16 x half> %acc, <vscale x 16 x half> %in)
  ret <vscale x 16 x half> %res
}

define <vscale x 4 x bfloat> @pr_nx4xbfloat(<vscale x 4 x bfloat> %in, <vscale x 4 x bfloat> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx4xbfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    ptrue p0.s
; NEON-NOBF16-NEXT:    bfadd z0.h, p0/m, z0.h, z1.h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx4xbfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    ptrue p0.s
; NEON-BF16-NEXT:    bfadd z0.h, p0/m, z0.h, z1.h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx4xbfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.s
; SVE-NEXT:    bfadd z0.h, p0/m, z0.h, z1.h
; SVE-NEXT:    ret
  %res = call <vscale x 4 x bfloat> @llvm.vector.partial.reduce.fadd(<vscale x 4 x bfloat> %acc, <vscale x 4 x bfloat> %in)
  ret <vscale x 4 x bfloat> %res
}

define <vscale x 8 x bfloat> @pr_nx8xbfloat(<vscale x 8 x bfloat> %in, <vscale x 8 x bfloat> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx8xbfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    bfadd z0.h, z1.h, z0.h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx8xbfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    bfadd z0.h, z1.h, z0.h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx8xbfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    bfadd z0.h, z1.h, z0.h
; SVE-NEXT:    ret
  %res = call <vscale x 8 x bfloat> @llvm.vector.partial.reduce.fadd(<vscale x 8 x bfloat> %acc, <vscale x 8 x bfloat> %in)
  ret <vscale x 8 x bfloat> %res
}

define <vscale x 16 x bfloat> @pr_nx16xbfloat(<vscale x 16 x bfloat> %in, <vscale x 16 x bfloat> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx16xbfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    bfadd z1.h, z3.h, z1.h
; NEON-NOBF16-NEXT:    bfadd z0.h, z2.h, z0.h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx16xbfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    bfadd z1.h, z3.h, z1.h
; NEON-BF16-NEXT:    bfadd z0.h, z2.h, z0.h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx16xbfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    bfadd z1.h, z3.h, z1.h
; SVE-NEXT:    bfadd z0.h, z2.h, z0.h
; SVE-NEXT:    ret
  %res = call <vscale x 16 x bfloat> @llvm.vector.partial.reduce.fadd(<vscale x 16 x bfloat> %acc, <vscale x 16 x bfloat> %in)
  ret <vscale x 16 x bfloat> %res
}

define <vscale x 2 x float> @pr_nx2xfloat(<vscale x 2 x float> %in, <vscale x 2 x float> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx2xfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    ptrue p0.d
; NEON-NOBF16-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx2xfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    ptrue p0.d
; NEON-BF16-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx2xfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.d
; SVE-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
; SVE-NEXT:    ret
  %res = call <vscale x 2 x float> @llvm.vector.partial.reduce.fadd(<vscale x 2 x float> %acc, <vscale x 2 x float> %in)
  ret <vscale x 2 x float> %res
}

define <vscale x 4 x float> @pr_nx4xfloat(<vscale x 4 x float> %in, <vscale x 4 x float> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx4xfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fadd z0.s, z1.s, z0.s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx4xfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fadd z0.s, z1.s, z0.s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx4xfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd z0.s, z1.s, z0.s
; SVE-NEXT:    ret
  %res = call <vscale x 4 x float> @llvm.vector.partial.reduce.fadd(<vscale x 4 x float> %acc, <vscale x 4 x float> %in)
  ret <vscale x 4 x float> %res
}

define <vscale x 8 x float> @pr_nx8xfloat(<vscale x 8 x float> %in, <vscale x 8 x float> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx8xfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fadd z1.s, z3.s, z1.s
; NEON-NOBF16-NEXT:    fadd z0.s, z2.s, z0.s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx8xfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fadd z1.s, z3.s, z1.s
; NEON-BF16-NEXT:    fadd z0.s, z2.s, z0.s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx8xfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd z1.s, z3.s, z1.s
; SVE-NEXT:    fadd z0.s, z2.s, z0.s
; SVE-NEXT:    ret
  %res = call <vscale x 8 x float> @llvm.vector.partial.reduce.fadd(<vscale x 8 x float> %acc, <vscale x 8 x float> %in)
  ret <vscale x 8 x float> %res
}

define <vscale x 2 x double> @pr_nx2xdouble(<vscale x 2 x double> %in, <vscale x 2 x double> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx2xdouble:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fadd z0.d, z1.d, z0.d
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx2xdouble:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fadd z0.d, z1.d, z0.d
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx2xdouble:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd z0.d, z1.d, z0.d
; SVE-NEXT:    ret
  %res = call <vscale x 2 x double> @llvm.vector.partial.reduce.fadd(<vscale x 2 x double> %acc, <vscale x 2 x double> %in)
  ret <vscale x 2 x double> %res
}

define <vscale x 4 x double> @pr_nx4xdouble(<vscale x 4 x double> %in, <vscale x 4 x double> %acc) #0 {
; NEON-NOBF16-LABEL: pr_nx4xdouble:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fadd z1.d, z3.d, z1.d
; NEON-NOBF16-NEXT:    fadd z0.d, z2.d, z0.d
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_nx4xdouble:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fadd z1.d, z3.d, z1.d
; NEON-BF16-NEXT:    fadd z0.d, z2.d, z0.d
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_nx4xdouble:
; SVE:       // %bb.0:
; SVE-NEXT:    fadd z1.d, z3.d, z1.d
; SVE-NEXT:    fadd z0.d, z2.d, z0.d
; SVE-NEXT:    ret
  %res = call <vscale x 4 x double> @llvm.vector.partial.reduce.fadd(<vscale x 4 x double> %acc, <vscale x 4 x double> %in)
  ret <vscale x 4 x double> %res
}

; Test combining with a multiply.

define <4 x i32> @pr_mla_4xi32(<4 x i32> %acc, <4 x i32> %lhs, <4 x i32> %rhs) #0 {
; NEON-NOBF16-LABEL: pr_mla_4xi32:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    mla v0.4s, v1.4s, v2.4s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_mla_4xi32:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    mla v0.4s, v1.4s, v2.4s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_mla_4xi32:
; SVE:       // %bb.0:
; SVE-NEXT:    mla v0.4s, v1.4s, v2.4s
; SVE-NEXT:    ret
  %mul = mul <4 x i32> %lhs, %rhs
  %res = call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> %acc, <4 x i32> %mul)
  ret <4 x i32> %res
}

define <vscale x 2 x double> @pr_fma_nx2xdouble(<vscale x 2 x double> %acc, <vscale x 2 x double> %lhs, <vscale x 2 x double> %rhs) #0 {
; NEON-NOBF16-LABEL: pr_fma_nx2xdouble:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    ptrue p0.d
; NEON-NOBF16-NEXT:    fmla z0.d, p0/m, z1.d, z2.d
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_fma_nx2xdouble:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    ptrue p0.d
; NEON-BF16-NEXT:    fmla z0.d, p0/m, z1.d, z2.d
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_fma_nx2xdouble:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.d
; SVE-NEXT:    fmla z0.d, p0/m, z1.d, z2.d
; SVE-NEXT:    ret
  %fmul = fmul contract <vscale x 2 x double> %lhs, %rhs
  %res = call contract <vscale x 2 x double> @llvm.vector.partial.reduce.fadd(<vscale x 2 x double> %acc, <vscale x 2 x double> %fmul)
  ret <vscale x 2 x double> %res
}

define <vscale x 4 x float> @pr_fma_nx4xfloat(<vscale x 4 x float> %acc, <vscale x 4 x float> %lhs, <vscale x 4 x float> %rhs) #0 {
; NEON-NOBF16-LABEL: pr_fma_nx4xfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    ptrue p0.s
; NEON-NOBF16-NEXT:    fmla z0.s, p0/m, z1.s, z2.s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_fma_nx4xfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    ptrue p0.s
; NEON-BF16-NEXT:    fmla z0.s, p0/m, z1.s, z2.s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_fma_nx4xfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.s
; SVE-NEXT:    fmla z0.s, p0/m, z1.s, z2.s
; SVE-NEXT:    ret
  %fmul = fmul contract <vscale x 4 x float> %lhs, %rhs
  %res = call contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd(<vscale x 4 x float> %acc, <vscale x 4 x float> %fmul)
  ret <vscale x 4 x float> %res
}

define <vscale x 8 x half> @pr_fma_nx8xhalf(<vscale x 8 x half> %acc, <vscale x 8 x half> %lhs, <vscale x 8 x half> %rhs) #0 {
; NEON-NOBF16-LABEL: pr_fma_nx8xhalf:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    ptrue p0.h
; NEON-NOBF16-NEXT:    fmla z0.h, p0/m, z1.h, z2.h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_fma_nx8xhalf:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    ptrue p0.h
; NEON-BF16-NEXT:    fmla z0.h, p0/m, z1.h, z2.h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_fma_nx8xhalf:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.h
; SVE-NEXT:    fmla z0.h, p0/m, z1.h, z2.h
; SVE-NEXT:    ret
  %fmul = fmul contract <vscale x 8 x half> %lhs, %rhs
  %res = call contract <vscale x 8 x half> @llvm.vector.partial.reduce.fadd(<vscale x 8 x half> %acc, <vscale x 8 x half> %fmul)
  ret <vscale x 8 x half> %res
}

define <vscale x 8 x bfloat> @pr_fma_nx8xbfloat(<vscale x 8 x bfloat> %acc, <vscale x 8 x bfloat> %lhs, <vscale x 8 x bfloat> %rhs) #0 {
; NEON-NOBF16-LABEL: pr_fma_nx8xbfloat:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    ptrue p0.h
; NEON-NOBF16-NEXT:    bfmla z0.h, p0/m, z1.h, z2.h
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_fma_nx8xbfloat:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    ptrue p0.h
; NEON-BF16-NEXT:    bfmla z0.h, p0/m, z1.h, z2.h
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_fma_nx8xbfloat:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.h
; SVE-NEXT:    bfmla z0.h, p0/m, z1.h, z2.h
; SVE-NEXT:    ret
  %fmul = fmul contract <vscale x 8 x bfloat> %lhs, %rhs
  %res = call contract <vscale x 8 x bfloat> @llvm.vector.partial.reduce.fadd(<vscale x 8 x bfloat> %acc, <vscale x 8 x bfloat> %fmul)
  ret <vscale x 8 x bfloat> %res
}

define <vscale x 2 x double> @pr_fma_nx2xdouble_contract_fmul_only(<vscale x 2 x double> %acc, <vscale x 2 x double> %lhs, <vscale x 2 x double> %rhs) #0 {
; NEON-NOBF16-LABEL: pr_fma_nx2xdouble_contract_fmul_only:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fmul z1.d, z1.d, z2.d
; NEON-NOBF16-NEXT:    fadd z0.d, z0.d, z1.d
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_fma_nx2xdouble_contract_fmul_only:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fmul z1.d, z1.d, z2.d
; NEON-BF16-NEXT:    fadd z0.d, z0.d, z1.d
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_fma_nx2xdouble_contract_fmul_only:
; SVE:       // %bb.0:
; SVE-NEXT:    fmul z1.d, z1.d, z2.d
; SVE-NEXT:    fadd z0.d, z0.d, z1.d
; SVE-NEXT:    ret
  %fmul = fmul contract <vscale x 2 x double> %lhs, %rhs
  %res = call <vscale x 2 x double> @llvm.vector.partial.reduce.fadd(<vscale x 2 x double> %acc, <vscale x 2 x double> %fmul)
  ret <vscale x 2 x double> %res
}

define <vscale x 2 x double> @pr_fma_nx2xdouble_non_contract(<vscale x 2 x double> %acc, <vscale x 2 x double> %lhs, <vscale x 2 x double> %rhs) #0 {
; NEON-NOBF16-LABEL: pr_fma_nx2xdouble_non_contract:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fmul z1.d, z1.d, z2.d
; NEON-NOBF16-NEXT:    fadd z0.d, z0.d, z1.d
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_fma_nx2xdouble_non_contract:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fmul z1.d, z1.d, z2.d
; NEON-BF16-NEXT:    fadd z0.d, z0.d, z1.d
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_fma_nx2xdouble_non_contract:
; SVE:       // %bb.0:
; SVE-NEXT:    fmul z1.d, z1.d, z2.d
; SVE-NEXT:    fadd z0.d, z0.d, z1.d
; SVE-NEXT:    ret
  %fmul = fmul <vscale x 2 x double> %lhs, %rhs
  %res = call <vscale x 2 x double> @llvm.vector.partial.reduce.fadd(<vscale x 2 x double> %acc, <vscale x 2 x double> %fmul)
  ret <vscale x 2 x double> %res
}

define <vscale x 4 x float> @pr_fma_nx4xfloat_wider_input(<vscale x 4 x float> %acc, <vscale x 8 x float> %lhs, <vscale x 8 x float> %rhs) #0 {
; NEON-NOBF16-LABEL: pr_fma_nx4xfloat_wider_input:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    ptrue p0.s
; NEON-NOBF16-NEXT:    fmla z0.s, p0/m, z1.s, z3.s
; NEON-NOBF16-NEXT:    fmla z0.s, p0/m, z2.s, z4.s
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_fma_nx4xfloat_wider_input:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    ptrue p0.s
; NEON-BF16-NEXT:    fmla z0.s, p0/m, z1.s, z3.s
; NEON-BF16-NEXT:    fmla z0.s, p0/m, z2.s, z4.s
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_fma_nx4xfloat_wider_input:
; SVE:       // %bb.0:
; SVE-NEXT:    ptrue p0.s
; SVE-NEXT:    fmla z0.s, p0/m, z1.s, z3.s
; SVE-NEXT:    fmla z0.s, p0/m, z2.s, z4.s
; SVE-NEXT:    ret
  %fmul = fmul contract <vscale x 8 x float> %lhs, %rhs
  %res = call contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd(<vscale x 4 x float> %acc, <vscale x 8 x float> %fmul)
  ret <vscale x 4 x float> %res
}

define <vscale x 2 x double> @pr_fma_nx2xdouble_second_use(<vscale x 2 x double> %acc, <vscale x 2 x double> %lhs, <vscale x 2 x double> %rhs) #0 {
; NEON-NOBF16-LABEL: pr_fma_nx2xdouble_second_use:
; NEON-NOBF16:       // %bb.0:
; NEON-NOBF16-NEXT:    fmul z1.d, z1.d, z2.d
; NEON-NOBF16-NEXT:    fadd z0.d, z0.d, z1.d
; NEON-NOBF16-NEXT:    fmul z0.d, z0.d, z1.d
; NEON-NOBF16-NEXT:    ret
;
; NEON-BF16-LABEL: pr_fma_nx2xdouble_second_use:
; NEON-BF16:       // %bb.0:
; NEON-BF16-NEXT:    fmul z1.d, z1.d, z2.d
; NEON-BF16-NEXT:    fadd z0.d, z0.d, z1.d
; NEON-BF16-NEXT:    fmul z0.d, z0.d, z1.d
; NEON-BF16-NEXT:    ret
;
; SVE-LABEL: pr_fma_nx2xdouble_second_use:
; SVE:       // %bb.0:
; SVE-NEXT:    fmul z1.d, z1.d, z2.d
; SVE-NEXT:    fadd z0.d, z0.d, z1.d
; SVE-NEXT:    fmul z0.d, z0.d, z1.d
; SVE-NEXT:    ret
  %fmul = fmul contract <vscale x 2 x double> %lhs, %rhs
  %pr.fadd = call contract <vscale x 2 x double> @llvm.vector.partial.reduce.fadd(<vscale x 2 x double> %acc, <vscale x 2 x double> %fmul)
  %res = fmul <vscale x 2 x double> %pr.fadd, %fmul
  ret <vscale x 2 x double> %res
}

attributes #0 = { nounwind "target-features"="+sve2,+bf16,+sve-b16b16" vscale_range(1,16) }