; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
; RUN: llc -mtriple=aarch64 -aarch64-enable-sink-fold=true -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple=aarch64 -aarch64-enable-sink-fold=true -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI

; CHECK_GI:        warning: Instruction selection used fallback path for mulv_v3i64

declare i8 @llvm.vector.reduce.mul.v2i8(<2 x i8>)
declare i8 @llvm.vector.reduce.mul.v3i8(<3 x i8>)
declare i8 @llvm.vector.reduce.mul.v4i8(<4 x i8>)
declare i8 @llvm.vector.reduce.mul.v8i8(<8 x i8>)
declare i8 @llvm.vector.reduce.mul.v16i8(<16 x i8>)
declare i8 @llvm.vector.reduce.mul.v32i8(<32 x i8>)
declare i16 @llvm.vector.reduce.mul.v2i16(<2 x i16>)
declare i16 @llvm.vector.reduce.mul.v3i16(<3 x i16>)
declare i16 @llvm.vector.reduce.mul.v4i16(<4 x i16>)
declare i16 @llvm.vector.reduce.mul.v8i16(<8 x i16>)
declare i16 @llvm.vector.reduce.mul.v16i16(<16 x i16>)
declare i32 @llvm.vector.reduce.mul.v2i32(<2 x i32>)
declare i32 @llvm.vector.reduce.mul.v3i32(<3 x i32>)
declare i32 @llvm.vector.reduce.mul.v4i32(<4 x i32>)
declare i32 @llvm.vector.reduce.mul.v8i32(<8 x i32>)
declare i64 @llvm.vector.reduce.mul.v2i64(<2 x i64>)
declare i64 @llvm.vector.reduce.mul.v3i64(<3 x i64>)
declare i64 @llvm.vector.reduce.mul.v4i64(<4 x i64>)
declare i128 @llvm.vector.reduce.mul.v2i128(<2 x i128>)

define i8 @mulv_v2i8(<2 x i8> %a) {
; CHECK-LABEL: mulv_v2i8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT:    mov w8, v0.s[1]
; CHECK-NEXT:    fmov w9, s0
; CHECK-NEXT:    mul w0, w9, w8
; CHECK-NEXT:    ret
entry:
  %arg1 = call i8 @llvm.vector.reduce.mul.v2i8(<2 x i8> %a)
  ret i8 %arg1
}

define i8 @mulv_v3i8(<3 x i8> %a) {
; CHECK-LABEL: mulv_v3i8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mul w8, w0, w1
; CHECK-NEXT:    mul w0, w8, w2
; CHECK-NEXT:    ret
entry:
  %arg1 = call i8 @llvm.vector.reduce.mul.v3i8(<3 x i8> %a)
  ret i8 %arg1
}

define i8 @mulv_v4i8(<4 x i8> %a) {
; CHECK-SD-LABEL: mulv_v4i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    umov w8, v0.h[1]
; CHECK-SD-NEXT:    umov w9, v0.h[0]
; CHECK-SD-NEXT:    umov w10, v0.h[2]
; CHECK-SD-NEXT:    mul w8, w9, w8
; CHECK-SD-NEXT:    umov w9, v0.h[3]
; CHECK-SD-NEXT:    mul w8, w8, w10
; CHECK-SD-NEXT:    mul w0, w8, w9
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulv_v4i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    umov w8, v0.h[0]
; CHECK-GI-NEXT:    umov w9, v0.h[1]
; CHECK-GI-NEXT:    umov w10, v0.h[2]
; CHECK-GI-NEXT:    umov w11, v0.h[3]
; CHECK-GI-NEXT:    mul w8, w8, w9
; CHECK-GI-NEXT:    mul w9, w10, w11
; CHECK-GI-NEXT:    mul w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %arg1 = call i8 @llvm.vector.reduce.mul.v4i8(<4 x i8> %a)
  ret i8 %arg1
}

define i8 @mulv_v8i8(<8 x i8> %a) {
; CHECK-SD-LABEL: mulv_v8i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    umov w8, v0.b[1]
; CHECK-SD-NEXT:    umov w9, v0.b[0]
; CHECK-SD-NEXT:    umov w10, v0.b[2]
; CHECK-SD-NEXT:    mul w8, w9, w8
; CHECK-SD-NEXT:    umov w9, v0.b[3]
; CHECK-SD-NEXT:    mul w8, w8, w10
; CHECK-SD-NEXT:    umov w10, v0.b[4]
; CHECK-SD-NEXT:    mul w8, w8, w9
; CHECK-SD-NEXT:    umov w9, v0.b[5]
; CHECK-SD-NEXT:    mul w8, w8, w10
; CHECK-SD-NEXT:    umov w10, v0.b[6]
; CHECK-SD-NEXT:    mul w8, w8, w9
; CHECK-SD-NEXT:    umov w9, v0.b[7]
; CHECK-SD-NEXT:    mul w8, w8, w10
; CHECK-SD-NEXT:    mul w0, w8, w9
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulv_v8i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    umov w8, v0.b[0]
; CHECK-GI-NEXT:    umov w9, v0.b[1]
; CHECK-GI-NEXT:    umov w10, v0.b[2]
; CHECK-GI-NEXT:    umov w11, v0.b[3]
; CHECK-GI-NEXT:    umov w12, v0.b[4]
; CHECK-GI-NEXT:    umov w13, v0.b[5]
; CHECK-GI-NEXT:    umov w14, v0.b[6]
; CHECK-GI-NEXT:    umov w15, v0.b[7]
; CHECK-GI-NEXT:    mul w8, w8, w9
; CHECK-GI-NEXT:    mul w9, w10, w11
; CHECK-GI-NEXT:    mul w10, w12, w13
; CHECK-GI-NEXT:    mul w11, w14, w15
; CHECK-GI-NEXT:    mul w8, w8, w9
; CHECK-GI-NEXT:    mul w9, w10, w11
; CHECK-GI-NEXT:    mul w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %arg1 = call i8 @llvm.vector.reduce.mul.v8i8(<8 x i8> %a)
  ret i8 %arg1
}

define i8 @mulv_v16i8(<16 x i8> %a) {
; CHECK-SD-LABEL: mulv_v16i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT:    mul v0.8b, v0.8b, v1.8b
; CHECK-SD-NEXT:    umov w8, v0.b[1]
; CHECK-SD-NEXT:    umov w9, v0.b[0]
; CHECK-SD-NEXT:    umov w10, v0.b[2]
; CHECK-SD-NEXT:    mul w8, w9, w8
; CHECK-SD-NEXT:    umov w9, v0.b[3]
; CHECK-SD-NEXT:    mul w8, w8, w10
; CHECK-SD-NEXT:    umov w10, v0.b[4]
; CHECK-SD-NEXT:    mul w8, w8, w9
; CHECK-SD-NEXT:    umov w9, v0.b[5]
; CHECK-SD-NEXT:    mul w8, w8, w10
; CHECK-SD-NEXT:    umov w10, v0.b[6]
; CHECK-SD-NEXT:    mul w8, w8, w9
; CHECK-SD-NEXT:    umov w9, v0.b[7]
; CHECK-SD-NEXT:    mul w8, w8, w10
; CHECK-SD-NEXT:    mul w0, w8, w9
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulv_v16i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov d1, v0.d[1]
; CHECK-GI-NEXT:    mul v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    umov w8, v0.b[0]
; CHECK-GI-NEXT:    umov w9, v0.b[1]
; CHECK-GI-NEXT:    umov w10, v0.b[2]
; CHECK-GI-NEXT:    umov w11, v0.b[3]
; CHECK-GI-NEXT:    umov w12, v0.b[4]
; CHECK-GI-NEXT:    umov w13, v0.b[5]
; CHECK-GI-NEXT:    umov w14, v0.b[6]
; CHECK-GI-NEXT:    umov w15, v0.b[7]
; CHECK-GI-NEXT:    mul w8, w8, w9
; CHECK-GI-NEXT:    mul w9, w10, w11
; CHECK-GI-NEXT:    mul w10, w12, w13
; CHECK-GI-NEXT:    mul w11, w14, w15
; CHECK-GI-NEXT:    mul w8, w8, w9
; CHECK-GI-NEXT:    mul w9, w10, w11
; CHECK-GI-NEXT:    mul w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %arg1 = call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> %a)
  ret i8 %arg1
}

define i8 @mulv_v32i8(<32 x i8> %a) {
; CHECK-SD-LABEL: mulv_v32i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mul v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT:    mul v0.8b, v0.8b, v1.8b
; CHECK-SD-NEXT:    umov w8, v0.b[1]
; CHECK-SD-NEXT:    umov w9, v0.b[0]
; CHECK-SD-NEXT:    umov w10, v0.b[2]
; CHECK-SD-NEXT:    mul w8, w9, w8
; CHECK-SD-NEXT:    umov w9, v0.b[3]
; CHECK-SD-NEXT:    mul w8, w8, w10
; CHECK-SD-NEXT:    umov w10, v0.b[4]
; CHECK-SD-NEXT:    mul w8, w8, w9
; CHECK-SD-NEXT:    umov w9, v0.b[5]
; CHECK-SD-NEXT:    mul w8, w8, w10
; CHECK-SD-NEXT:    umov w10, v0.b[6]
; CHECK-SD-NEXT:    mul w8, w8, w9
; CHECK-SD-NEXT:    umov w9, v0.b[7]
; CHECK-SD-NEXT:    mul w8, w8, w10
; CHECK-SD-NEXT:    mul w0, w8, w9
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulv_v32i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov d2, v0.d[1]
; CHECK-GI-NEXT:    mov d3, v1.d[1]
; CHECK-GI-NEXT:    mul v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    mul v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    mul v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    umov w8, v0.b[0]
; CHECK-GI-NEXT:    umov w9, v0.b[1]
; CHECK-GI-NEXT:    umov w10, v0.b[2]
; CHECK-GI-NEXT:    umov w11, v0.b[3]
; CHECK-GI-NEXT:    umov w12, v0.b[4]
; CHECK-GI-NEXT:    umov w13, v0.b[5]
; CHECK-GI-NEXT:    umov w14, v0.b[6]
; CHECK-GI-NEXT:    umov w15, v0.b[7]
; CHECK-GI-NEXT:    mul w8, w8, w9
; CHECK-GI-NEXT:    mul w9, w10, w11
; CHECK-GI-NEXT:    mul w10, w12, w13
; CHECK-GI-NEXT:    mul w11, w14, w15
; CHECK-GI-NEXT:    mul w8, w8, w9
; CHECK-GI-NEXT:    mul w9, w10, w11
; CHECK-GI-NEXT:    mul w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %arg1 = call i8 @llvm.vector.reduce.mul.v32i8(<32 x i8> %a)
  ret i8 %arg1
}

define i16 @mulv_v2i16(<2 x i16> %a) {
; CHECK-LABEL: mulv_v2i16:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT:    mov w8, v0.s[1]
; CHECK-NEXT:    fmov w9, s0
; CHECK-NEXT:    mul w0, w9, w8
; CHECK-NEXT:    ret
entry:
  %arg1 = call i16 @llvm.vector.reduce.mul.v2i16(<2 x i16> %a)
  ret i16 %arg1
}

define i16 @mulv_v3i16(<3 x i16> %a) {
; CHECK-SD-LABEL: mulv_v3i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    umov w8, v0.h[1]
; CHECK-SD-NEXT:    umov w9, v0.h[0]
; CHECK-SD-NEXT:    umov w10, v0.h[2]
; CHECK-SD-NEXT:    mul w8, w9, w8
; CHECK-SD-NEXT:    mul w0, w8, w10
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulv_v3i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    umov w8, v0.h[0]
; CHECK-GI-NEXT:    umov w9, v0.h[1]
; CHECK-GI-NEXT:    umov w10, v0.h[2]
; CHECK-GI-NEXT:    mul w8, w8, w9
; CHECK-GI-NEXT:    mul w0, w8, w10
; CHECK-GI-NEXT:    ret
entry:
  %arg1 = call i16 @llvm.vector.reduce.mul.v3i16(<3 x i16> %a)
  ret i16 %arg1
}

define i16 @mulv_v4i16(<4 x i16> %a) {
; CHECK-SD-LABEL: mulv_v4i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    umov w8, v0.h[1]
; CHECK-SD-NEXT:    umov w9, v0.h[0]
; CHECK-SD-NEXT:    umov w10, v0.h[2]
; CHECK-SD-NEXT:    mul w8, w9, w8
; CHECK-SD-NEXT:    umov w9, v0.h[3]
; CHECK-SD-NEXT:    mul w8, w8, w10
; CHECK-SD-NEXT:    mul w0, w8, w9
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulv_v4i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    umov w8, v0.h[0]
; CHECK-GI-NEXT:    umov w9, v0.h[1]
; CHECK-GI-NEXT:    umov w10, v0.h[2]
; CHECK-GI-NEXT:    umov w11, v0.h[3]
; CHECK-GI-NEXT:    mul w8, w8, w9
; CHECK-GI-NEXT:    mul w9, w10, w11
; CHECK-GI-NEXT:    mul w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %arg1 = call i16 @llvm.vector.reduce.mul.v4i16(<4 x i16> %a)
  ret i16 %arg1
}

define i16 @mulv_v8i16(<8 x i16> %a) {
; CHECK-SD-LABEL: mulv_v8i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT:    mul v0.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    umov w8, v0.h[1]
; CHECK-SD-NEXT:    umov w9, v0.h[0]
; CHECK-SD-NEXT:    umov w10, v0.h[2]
; CHECK-SD-NEXT:    mul w8, w9, w8
; CHECK-SD-NEXT:    umov w9, v0.h[3]
; CHECK-SD-NEXT:    mul w8, w8, w10
; CHECK-SD-NEXT:    mul w0, w8, w9
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulv_v8i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov d1, v0.d[1]
; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    umov w8, v0.h[0]
; CHECK-GI-NEXT:    umov w9, v0.h[1]
; CHECK-GI-NEXT:    umov w10, v0.h[2]
; CHECK-GI-NEXT:    umov w11, v0.h[3]
; CHECK-GI-NEXT:    mul w8, w8, w9
; CHECK-GI-NEXT:    mul w9, w10, w11
; CHECK-GI-NEXT:    mul w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %arg1 = call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> %a)
  ret i16 %arg1
}

define i16 @mulv_v16i16(<16 x i16> %a) {
; CHECK-SD-LABEL: mulv_v16i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mul v0.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT:    mul v0.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    umov w8, v0.h[1]
; CHECK-SD-NEXT:    umov w9, v0.h[0]
; CHECK-SD-NEXT:    umov w10, v0.h[2]
; CHECK-SD-NEXT:    mul w8, w9, w8
; CHECK-SD-NEXT:    umov w9, v0.h[3]
; CHECK-SD-NEXT:    mul w8, w8, w10
; CHECK-SD-NEXT:    mul w0, w8, w9
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulv_v16i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov d2, v0.d[1]
; CHECK-GI-NEXT:    mov d3, v1.d[1]
; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v2.4h
; CHECK-GI-NEXT:    mul v1.4h, v1.4h, v3.4h
; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    umov w8, v0.h[0]
; CHECK-GI-NEXT:    umov w9, v0.h[1]
; CHECK-GI-NEXT:    umov w10, v0.h[2]
; CHECK-GI-NEXT:    umov w11, v0.h[3]
; CHECK-GI-NEXT:    mul w8, w8, w9
; CHECK-GI-NEXT:    mul w9, w10, w11
; CHECK-GI-NEXT:    mul w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %arg1 = call i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> %a)
  ret i16 %arg1
}

define i32 @mulv_v2i32(<2 x i32> %a) {
; CHECK-LABEL: mulv_v2i32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT:    mov w8, v0.s[1]
; CHECK-NEXT:    fmov w9, s0
; CHECK-NEXT:    mul w0, w9, w8
; CHECK-NEXT:    ret
entry:
  %arg1 = call i32 @llvm.vector.reduce.mul.v2i32(<2 x i32> %a)
  ret i32 %arg1
}

define i32 @mulv_v3i32(<3 x i32> %a) {
; CHECK-LABEL: mulv_v3i32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov v1.16b, v0.16b
; CHECK-NEXT:    mov w8, #1 // =0x1
; CHECK-NEXT:    mov v1.s[3], w8
; CHECK-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
; CHECK-NEXT:    mul v0.2s, v0.2s, v1.2s
; CHECK-NEXT:    mov w8, v0.s[1]
; CHECK-NEXT:    fmov w9, s0
; CHECK-NEXT:    mul w0, w9, w8
; CHECK-NEXT:    ret
entry:
  %arg1 = call i32 @llvm.vector.reduce.mul.v3i32(<3 x i32> %a)
  ret i32 %arg1
}

define i32 @mulv_v4i32(<4 x i32> %a) {
; CHECK-SD-LABEL: mulv_v4i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.2s
; CHECK-SD-NEXT:    mov w8, v0.s[1]
; CHECK-SD-NEXT:    fmov w9, s0
; CHECK-SD-NEXT:    mul w0, w9, w8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulv_v4i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov d1, v0.d[1]
; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    mov w8, v0.s[1]
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    mul w0, w9, w8
; CHECK-GI-NEXT:    ret
entry:
  %arg1 = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %a)
  ret i32 %arg1
}

define i32 @mulv_v8i32(<8 x i32> %a) {
; CHECK-SD-LABEL: mulv_v8i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mul v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.2s
; CHECK-SD-NEXT:    mov w8, v0.s[1]
; CHECK-SD-NEXT:    fmov w9, s0
; CHECK-SD-NEXT:    mul w0, w9, w8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulv_v8i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov d2, v0.d[1]
; CHECK-GI-NEXT:    mov d3, v1.d[1]
; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v2.2s
; CHECK-GI-NEXT:    mul v1.2s, v1.2s, v3.2s
; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    mov w8, v0.s[1]
; CHECK-GI-NEXT:    fmov w9, s0
; CHECK-GI-NEXT:    mul w0, w9, w8
; CHECK-GI-NEXT:    ret
entry:
  %arg1 = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> %a)
  ret i32 %arg1
}

define i64 @mulv_v2i64(<2 x i64> %a) {
; CHECK-LABEL: mulv_v2i64:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov x8, v0.d[1]
; CHECK-NEXT:    fmov x9, d0
; CHECK-NEXT:    mul x0, x9, x8
; CHECK-NEXT:    ret
entry:
  %arg1 = call i64 @llvm.vector.reduce.mul.v2i64(<2 x i64> %a)
  ret i64 %arg1
}

define i64 @mulv_v3i64(<3 x i64> %a) {
; CHECK-SD-LABEL: mulv_v3i64:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    fmov x8, d2
; CHECK-SD-NEXT:    fmov x9, d0
; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-SD-NEXT:    mul x8, x9, x8
; CHECK-SD-NEXT:    fmov x9, d1
; CHECK-SD-NEXT:    mul x0, x9, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulv_v3i64:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    mul x8, x8, x9
; CHECK-GI-NEXT:    fmov x9, d2
; CHECK-GI-NEXT:    mul x0, x8, x9
; CHECK-GI-NEXT:    ret
entry:
  %arg1 = call i64 @llvm.vector.reduce.mul.v3i64(<3 x i64> %a)
  ret i64 %arg1
}

define i64 @mulv_v4i64(<4 x i64> %a) {
; CHECK-SD-LABEL: mulv_v4i64:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mov x8, v1.d[1]
; CHECK-SD-NEXT:    mov x9, v0.d[1]
; CHECK-SD-NEXT:    fmov x10, d0
; CHECK-SD-NEXT:    mul x8, x9, x8
; CHECK-SD-NEXT:    fmov x9, d1
; CHECK-SD-NEXT:    mul x9, x10, x9
; CHECK-SD-NEXT:    mul x0, x9, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulv_v4i64:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov x8, v0.d[1]
; CHECK-GI-NEXT:    fmov x10, d0
; CHECK-GI-NEXT:    mov x9, v1.d[1]
; CHECK-GI-NEXT:    mul x8, x10, x8
; CHECK-GI-NEXT:    fmov x10, d1
; CHECK-GI-NEXT:    mul x9, x10, x9
; CHECK-GI-NEXT:    mul x0, x8, x9
; CHECK-GI-NEXT:    ret
entry:
  %arg1 = call i64 @llvm.vector.reduce.mul.v4i64(<4 x i64> %a)
  ret i64 %arg1
}

define i128 @mulv_v2i128(<2 x i128> %a) {
; CHECK-SD-LABEL: mulv_v2i128:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    umulh x8, x0, x2
; CHECK-SD-NEXT:    madd x8, x0, x3, x8
; CHECK-SD-NEXT:    mul x0, x0, x2
; CHECK-SD-NEXT:    madd x1, x1, x2, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: mulv_v2i128:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mul x9, x0, x3
; CHECK-GI-NEXT:    mul x8, x0, x2
; CHECK-GI-NEXT:    umulh x10, x0, x2
; CHECK-GI-NEXT:    madd x9, x1, x2, x9
; CHECK-GI-NEXT:    mov x0, x8
; CHECK-GI-NEXT:    add x1, x9, x10
; CHECK-GI-NEXT:    ret
entry:
  %arg1 = call i128 @llvm.vector.reduce.mul.v2i128(<2 x i128> %a)
  ret i128 %arg1
}