; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3           | FileCheck %s --check-prefixes=SSSE3-SLOW
; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,fast-hops | FileCheck %s --check-prefixes=SSSE3-FAST
; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx             | FileCheck %s --check-prefixes=AVX-SLOW,AVX1-SLOW
; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops   | FileCheck %s --check-prefixes=AVX-FAST,AVX1-FAST
; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2            | FileCheck %s --check-prefixes=AVX-SLOW,AVX2-SLOW
; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2,fast-hops  | FileCheck %s --check-prefixes=AVX-FAST,AVX2-FAST

; Vectorized Pairwise Sum Reductions
; e.g.
; inline STYPE sum(VTYPE x) {
;   return (x[0] + x[1]) + (x[2] + x[3]);
; }
;
; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) {
;   return (VTYPE) { sum( A0 ), sum( A1 ), sum( A2 ), sum( A3 ) };
; }

define <4 x float> @pair_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {
; SSSE3-SLOW-LABEL: pair_sum_v4f32_v4f32:
; SSSE3-SLOW:       # %bb.0:
; SSSE3-SLOW-NEXT:    haddps %xmm0, %xmm0
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm4, %xmm0
; SSSE3-SLOW-NEXT:    haddps %xmm1, %xmm1
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm4
; SSSE3-SLOW-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
; SSSE3-SLOW-NEXT:    haddps %xmm2, %xmm2
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm2, %xmm1
; SSSE3-SLOW-NEXT:    haddps %xmm3, %xmm3
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm2 = xmm3[1,1,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm3, %xmm2
; SSSE3-SLOW-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm1[0]
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
; SSSE3-SLOW-NEXT:    retq
;
; SSSE3-FAST-LABEL: pair_sum_v4f32_v4f32:
; SSSE3-FAST:       # %bb.0:
; SSSE3-FAST-NEXT:    haddps %xmm1, %xmm0
; SSSE3-FAST-NEXT:    haddps %xmm3, %xmm2
; SSSE3-FAST-NEXT:    haddps %xmm2, %xmm0
; SSSE3-FAST-NEXT:    retq
;
; AVX1-SLOW-LABEL: pair_sum_v4f32_v4f32:
; AVX1-SLOW:       # %bb.0:
; AVX1-SLOW-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
; AVX1-SLOW-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
; AVX1-SLOW-NEXT:    vaddps %xmm4, %xmm0, %xmm0
; AVX1-SLOW-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
; AVX1-SLOW-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
; AVX1-SLOW-NEXT:    vaddps %xmm4, %xmm1, %xmm1
; AVX1-SLOW-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; AVX1-SLOW-NEXT:    vhaddps %xmm2, %xmm2, %xmm1
; AVX1-SLOW-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
; AVX1-SLOW-NEXT:    vaddps %xmm2, %xmm1, %xmm1
; AVX1-SLOW-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX1-SLOW-NEXT:    vhaddps %xmm3, %xmm3, %xmm1
; AVX1-SLOW-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
; AVX1-SLOW-NEXT:    vaddps %xmm2, %xmm1, %xmm1
; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
; AVX1-SLOW-NEXT:    retq
;
; AVX-FAST-LABEL: pair_sum_v4f32_v4f32:
; AVX-FAST:       # %bb.0:
; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT:    vhaddps %xmm3, %xmm2, %xmm1
; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT:    retq
;
; AVX2-SLOW-LABEL: pair_sum_v4f32_v4f32:
; AVX2-SLOW:       # %bb.0:
; AVX2-SLOW-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
; AVX2-SLOW-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
; AVX2-SLOW-NEXT:    vaddps %xmm4, %xmm0, %xmm0
; AVX2-SLOW-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
; AVX2-SLOW-NEXT:    vaddps %xmm4, %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX2-SLOW-NEXT:    vhaddps %xmm2, %xmm2, %xmm1
; AVX2-SLOW-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
; AVX2-SLOW-NEXT:    vaddps %xmm2, %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3]
; AVX2-SLOW-NEXT:    vhaddps %xmm3, %xmm3, %xmm1
; AVX2-SLOW-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
; AVX2-SLOW-NEXT:    vaddps %xmm2, %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
; AVX2-SLOW-NEXT:    retq
  %5 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2>
  %6 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3>
  %7 = fadd <2 x float> %5, %6
  %8 = shufflevector <2 x float> %7, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
  %9 = fadd <2 x float> %7, %8
  %10 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 0, i32 2>
  %11 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 1, i32 3>
  %12 = fadd <2 x float> %10, %11
  %13 = shufflevector <2 x float> %12, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
  %14 = fadd <2 x float> %12, %13
  %15 = shufflevector <2 x float> %9, <2 x float> %14, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
  %16 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 0, i32 2>
  %17 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 1, i32 3>
  %18 = fadd <2 x float> %16, %17
  %19 = shufflevector <2 x float> %18, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
  %20 = fadd <2 x float> %18, %19
  %21 = shufflevector <2 x float> %20, <2 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
  %22 = shufflevector <4 x float> %15, <4 x float> %21, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
  %23 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 0, i32 2>
  %24 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 1, i32 3>
  %25 = fadd <2 x float> %23, %24
  %26 = shufflevector <2 x float> %25, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
  %27 = fadd <2 x float> %25, %26
  %28 = shufflevector <2 x float> %27, <2 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
  %29 = shufflevector <4 x float> %22, <4 x float> %28, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
  ret <4 x float> %29
}

define <4 x i32> @pair_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) {
; SSSE3-SLOW-LABEL: pair_sum_v4i32_v4i32:
; SSSE3-SLOW:       # %bb.0:
; SSSE3-SLOW-NEXT:    phaddd %xmm0, %xmm0
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
; SSSE3-SLOW-NEXT:    paddd %xmm4, %xmm0
; SSSE3-SLOW-NEXT:    phaddd %xmm1, %xmm1
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm4
; SSSE3-SLOW-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
; SSSE3-SLOW-NEXT:    phaddd %xmm2, %xmm2
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
; SSSE3-SLOW-NEXT:    paddd %xmm2, %xmm1
; SSSE3-SLOW-NEXT:    phaddd %xmm3, %xmm3
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]
; SSSE3-SLOW-NEXT:    paddd %xmm3, %xmm2
; SSSE3-SLOW-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
; SSSE3-SLOW-NEXT:    retq
;
; SSSE3-FAST-LABEL: pair_sum_v4i32_v4i32:
; SSSE3-FAST:       # %bb.0:
; SSSE3-FAST-NEXT:    phaddd %xmm3, %xmm2
; SSSE3-FAST-NEXT:    phaddd %xmm1, %xmm0
; SSSE3-FAST-NEXT:    phaddd %xmm2, %xmm0
; SSSE3-FAST-NEXT:    retq
;
; AVX1-SLOW-LABEL: pair_sum_v4i32_v4i32:
; AVX1-SLOW:       # %bb.0:
; AVX1-SLOW-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
; AVX1-SLOW-NEXT:    vpaddd %xmm4, %xmm0, %xmm0
; AVX1-SLOW-NEXT:    vphaddd %xmm1, %xmm1, %xmm1
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
; AVX1-SLOW-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
; AVX1-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; AVX1-SLOW-NEXT:    vphaddd %xmm2, %xmm2, %xmm1
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
; AVX1-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX1-SLOW-NEXT:    vphaddd %xmm3, %xmm3, %xmm1
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
; AVX1-SLOW-NEXT:    retq
;
; AVX-FAST-LABEL: pair_sum_v4i32_v4i32:
; AVX-FAST:       # %bb.0:
; AVX-FAST-NEXT:    vphaddd %xmm3, %xmm2, %xmm2
; AVX-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT:    vphaddd %xmm2, %xmm0, %xmm0
; AVX-FAST-NEXT:    retq
;
; AVX2-SLOW-LABEL: pair_sum_v4i32_v4i32:
; AVX2-SLOW:       # %bb.0:
; AVX2-SLOW-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
; AVX2-SLOW-NEXT:    vpaddd %xmm4, %xmm0, %xmm0
; AVX2-SLOW-NEXT:    vphaddd %xmm1, %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
; AVX2-SLOW-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX2-SLOW-NEXT:    vphaddd %xmm2, %xmm2, %xmm1
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
; AVX2-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3]
; AVX2-SLOW-NEXT:    vphaddd %xmm3, %xmm3, %xmm1
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
; AVX2-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
; AVX2-SLOW-NEXT:    retq
  %5 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
  %6 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
  %7 = add <2 x i32> %5, %6
  %8 = shufflevector <2 x i32> %7, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
  %9 = add <2 x i32> %7, %8
  %10 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
  %11 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
  %12 = add <2 x i32> %10, %11
  %13 = shufflevector <2 x i32> %12, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
  %14 = add <2 x i32> %12, %13
  %15 = shufflevector <2 x i32> %9, <2 x i32> %14, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
  %16 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
  %17 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
  %18 = add <2 x i32> %16, %17
  %19 = shufflevector <2 x i32> %18, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
  %20 = add <2 x i32> %18, %19
  %21 = shufflevector <2 x i32> %20, <2 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
  %22 = shufflevector <4 x i32> %15, <4 x i32> %21, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
  %23 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
  %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
  %25 = add <2 x i32> %23, %24
  %26 = shufflevector <2 x i32> %25, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
  %27 = add <2 x i32> %25, %26
  %28 = shufflevector <2 x i32> %27, <2 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
  %29 = shufflevector <4 x i32> %22, <4 x i32> %28, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
  ret <4 x i32> %29
}

define <8 x float> @pair_sum_v8f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3, <4 x float> %4, <4 x float> %5, <4 x float> %6, <4 x float> %7) {
; SSSE3-SLOW-LABEL: pair_sum_v8f32_v4f32:
; SSSE3-SLOW:       # %bb.0:
; SSSE3-SLOW-NEXT:    movaps %xmm3, %xmm8
; SSSE3-SLOW-NEXT:    haddps %xmm0, %xmm0
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm3, %xmm0
; SSSE3-SLOW-NEXT:    haddps %xmm1, %xmm1
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm3
; SSSE3-SLOW-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
; SSSE3-SLOW-NEXT:    movaps %xmm2, %xmm1
; SSSE3-SLOW-NEXT:    haddps %xmm8, %xmm1
; SSSE3-SLOW-NEXT:    haddps %xmm2, %xmm8
; SSSE3-SLOW-NEXT:    haddps %xmm4, %xmm5
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm5[2,0]
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm8 = xmm8[3,1],xmm5[3,1]
; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm8
; SSSE3-SLOW-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm8[0]
; SSSE3-SLOW-NEXT:    haddps %xmm6, %xmm6
; SSSE3-SLOW-NEXT:    haddps %xmm7, %xmm7
; SSSE3-SLOW-NEXT:    haddps %xmm7, %xmm6
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,3],xmm6[0,2]
; SSSE3-SLOW-NEXT:    movaps %xmm8, %xmm1
; SSSE3-SLOW-NEXT:    retq
;
; SSSE3-FAST-LABEL: pair_sum_v8f32_v4f32:
; SSSE3-FAST:       # %bb.0:
; SSSE3-FAST-NEXT:    haddps %xmm1, %xmm0
; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm0
; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm1
; SSSE3-FAST-NEXT:    haddps %xmm3, %xmm1
; SSSE3-FAST-NEXT:    haddps %xmm2, %xmm3
; SSSE3-FAST-NEXT:    haddps %xmm4, %xmm5
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,2],xmm5[2,0]
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1],xmm5[3,1]
; SSSE3-FAST-NEXT:    addps %xmm1, %xmm3
; SSSE3-FAST-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]
; SSSE3-FAST-NEXT:    haddps %xmm6, %xmm6
; SSSE3-FAST-NEXT:    haddps %xmm7, %xmm7
; SSSE3-FAST-NEXT:    haddps %xmm7, %xmm6
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[2,3],xmm6[0,2]
; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm1
; SSSE3-FAST-NEXT:    retq
;
; AVX1-SLOW-LABEL: pair_sum_v8f32_v4f32:
; AVX1-SLOW:       # %bb.0:
; AVX1-SLOW-NEXT:    vhaddps %xmm0, %xmm0, %xmm8
; AVX1-SLOW-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm8[1,1,3,3]
; AVX1-SLOW-NEXT:    vaddps %xmm0, %xmm8, %xmm8
; AVX1-SLOW-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
; AVX1-SLOW-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
; AVX1-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
; AVX1-SLOW-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
; AVX1-SLOW-NEXT:    vhaddps %xmm3, %xmm2, %xmm1
; AVX1-SLOW-NEXT:    vhaddps %xmm2, %xmm2, %xmm2
; AVX1-SLOW-NEXT:    vhaddps %xmm3, %xmm3, %xmm3
; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[1],xmm3[1],zero,zero
; AVX1-SLOW-NEXT:    vhaddps %xmm4, %xmm4, %xmm3
; AVX1-SLOW-NEXT:    vhaddps %xmm5, %xmm5, %xmm4
; AVX1-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,1]
; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0]
; AVX1-SLOW-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[1,3]
; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[1]
; AVX1-SLOW-NEXT:    vaddps %xmm2, %xmm1, %xmm1
; AVX1-SLOW-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX1-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
; AVX1-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
; AVX1-SLOW-NEXT:    vhaddps %xmm6, %xmm6, %xmm2
; AVX1-SLOW-NEXT:    vhaddps %xmm7, %xmm7, %xmm3
; AVX1-SLOW-NEXT:    vhaddps %xmm3, %xmm2, %xmm2
; AVX1-SLOW-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3]
; AVX1-SLOW-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-SLOW-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
; AVX1-SLOW-NEXT:    retq
;
; AVX1-FAST-LABEL: pair_sum_v8f32_v4f32:
; AVX1-FAST:       # %bb.0:
; AVX1-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT:    vhaddps %xmm3, %xmm2, %xmm1
; AVX1-FAST-NEXT:    vhaddps %xmm2, %xmm2, %xmm2
; AVX1-FAST-NEXT:    vhaddps %xmm3, %xmm3, %xmm3
; AVX1-FAST-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[1],xmm3[1],zero,zero
; AVX1-FAST-NEXT:    vhaddps %xmm4, %xmm4, %xmm3
; AVX1-FAST-NEXT:    vhaddps %xmm5, %xmm5, %xmm4
; AVX1-FAST-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,1]
; AVX1-FAST-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0]
; AVX1-FAST-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[1,3]
; AVX1-FAST-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[1]
; AVX1-FAST-NEXT:    vaddps %xmm2, %xmm1, %xmm1
; AVX1-FAST-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX1-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
; AVX1-FAST-NEXT:    vhaddps %xmm7, %xmm6, %xmm2
; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm2, %xmm2
; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-FAST-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
; AVX1-FAST-NEXT:    retq
;
; AVX2-SLOW-LABEL: pair_sum_v8f32_v4f32:
; AVX2-SLOW:       # %bb.0:
; AVX2-SLOW-NEXT:    vhaddps %xmm0, %xmm0, %xmm8
; AVX2-SLOW-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm8[1,1,3,3]
; AVX2-SLOW-NEXT:    vaddps %xmm0, %xmm8, %xmm8
; AVX2-SLOW-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
; AVX2-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
; AVX2-SLOW-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
; AVX2-SLOW-NEXT:    vhaddps %xmm4, %xmm4, %xmm1
; AVX2-SLOW-NEXT:    vhaddps %xmm5, %xmm5, %xmm4
; AVX2-SLOW-NEXT:    vhaddps %xmm3, %xmm2, %xmm2
; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3]
; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1]
; AVX2-SLOW-NEXT:    vaddps %xmm1, %xmm3, %xmm1
; AVX2-SLOW-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX2-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
; AVX2-SLOW-NEXT:    vhaddps %xmm6, %xmm6, %xmm2
; AVX2-SLOW-NEXT:    vhaddps %xmm7, %xmm7, %xmm3
; AVX2-SLOW-NEXT:    vhaddps %xmm3, %xmm2, %xmm2
; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3]
; AVX2-SLOW-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX2-SLOW-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
; AVX2-SLOW-NEXT:    retq
;
; AVX2-FAST-LABEL: pair_sum_v8f32_v4f32:
; AVX2-FAST:       # %bb.0:
; AVX2-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
; AVX2-FAST-NEXT:    vhaddps %xmm4, %xmm4, %xmm1
; AVX2-FAST-NEXT:    vhaddps %xmm5, %xmm5, %xmm4
; AVX2-FAST-NEXT:    vhaddps %xmm3, %xmm2, %xmm2
; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3]
; AVX2-FAST-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; AVX2-FAST-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1]
; AVX2-FAST-NEXT:    vaddps %xmm1, %xmm3, %xmm1
; AVX2-FAST-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX2-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
; AVX2-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
; AVX2-FAST-NEXT:    vhaddps %xmm7, %xmm6, %xmm2
; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm2, %xmm2
; AVX2-FAST-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX2-FAST-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
; AVX2-FAST-NEXT:    retq
  %9 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2>
  %10 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3>
  %11 = fadd <2 x float> %9, %10
  %12 = shufflevector <2 x float> %11, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
  %13 = fadd <2 x float> %11, %12
  %14 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 0, i32 2>
  %15 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 1, i32 3>
  %16 = fadd <2 x float> %14, %15
  %17 = shufflevector <2 x float> %16, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
  %18 = fadd <2 x float> %16, %17
  %19 = shufflevector <2 x float> %13, <2 x float> %18, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %20 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 0, i32 2>
  %21 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 1, i32 3>
  %22 = fadd <2 x float> %20, %21
  %23 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 0, i32 2>
  %24 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 1, i32 3>
  %25 = fadd <2 x float> %23, %24
  %26 = shufflevector <4 x float> %4, <4 x float> poison, <2 x i32> <i32 0, i32 2>
  %27 = shufflevector <4 x float> %4, <4 x float> poison, <2 x i32> <i32 1, i32 3>
  %28 = fadd <2 x float> %26, %27
  %29 = shufflevector <2 x float> %28, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  %30 = shufflevector <4 x float> %5, <4 x float> poison, <2 x i32> <i32 0, i32 2>
  %31 = shufflevector <4 x float> %5, <4 x float> poison, <2 x i32> <i32 1, i32 3>
  %32 = fadd <2 x float> %30, %31
  %33 = shufflevector <2 x float> %32, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  %34 = shufflevector <2 x float> %22, <2 x float> %25, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
  %35 = shufflevector <4 x float> %34, <4 x float> %29, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
  %36 = shufflevector <4 x float> %35, <4 x float> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
  %37 = shufflevector <2 x float> %22, <2 x float> %25, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
  %38 = shufflevector <4 x float> %37, <4 x float> %29, <4 x i32> <i32 0, i32 1, i32 5, i32 undef>
  %39 = shufflevector <4 x float> %38, <4 x float> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 5>
  %40 = fadd <4 x float> %36, %39
  %41 = shufflevector <4 x float> %40, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
  %42 = shufflevector <8 x float> %19, <8 x float> %41, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef>
  %43 = shufflevector <4 x float> %6, <4 x float> poison, <2 x i32> <i32 0, i32 2>
  %44 = shufflevector <4 x float> %6, <4 x float> poison, <2 x i32> <i32 1, i32 3>
  %45 = fadd <2 x float> %43, %44
  %46 = shufflevector <4 x float> %7, <4 x float> poison, <2 x i32> <i32 0, i32 2>
  %47 = shufflevector <4 x float> %7, <4 x float> poison, <2 x i32> <i32 1, i32 3>
  %48 = fadd <2 x float> %46, %47
  %49 = shufflevector <2 x float> %45, <2 x float> %48, <2 x i32> <i32 0, i32 2>
  %50 = shufflevector <2 x float> %45, <2 x float> %48, <2 x i32> <i32 1, i32 3>
  %51 = fadd <2 x float> %49, %50
  %52 = shufflevector <2 x float> %51, <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %53 = shufflevector <8 x float> %42, <8 x float> %52, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
  ret <8 x float> %53
}

define <8 x i32> @pair_sum_v8i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3, <4 x i32> %4, <4 x i32> %5, <4 x i32> %6, <4 x i32> %7) {
; SSSE3-SLOW-LABEL: pair_sum_v8i32_v4i32:
; SSSE3-SLOW:       # %bb.0:
; SSSE3-SLOW-NEXT:    movdqa %xmm2, %xmm8
; SSSE3-SLOW-NEXT:    phaddd %xmm0, %xmm0
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
; SSSE3-SLOW-NEXT:    paddd %xmm2, %xmm0
; SSSE3-SLOW-NEXT:    phaddd %xmm1, %xmm1
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm2
; SSSE3-SLOW-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; SSSE3-SLOW-NEXT:    phaddd %xmm3, %xmm8
; SSSE3-SLOW-NEXT:    movdqa %xmm5, %xmm1
; SSSE3-SLOW-NEXT:    phaddd %xmm4, %xmm5
; SSSE3-SLOW-NEXT:    phaddd %xmm4, %xmm4
; SSSE3-SLOW-NEXT:    phaddd %xmm1, %xmm1
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm4[1,1]
; SSSE3-SLOW-NEXT:    movdqa %xmm8, %xmm2
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm5[2,0]
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm8 = xmm8[1,3],xmm1[2,0]
; SSSE3-SLOW-NEXT:    paddd %xmm2, %xmm8
; SSSE3-SLOW-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm8[0]
; SSSE3-SLOW-NEXT:    phaddd %xmm6, %xmm6
; SSSE3-SLOW-NEXT:    phaddd %xmm7, %xmm7
; SSSE3-SLOW-NEXT:    phaddd %xmm7, %xmm6
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm8 = xmm8[2,3],xmm6[0,2]
; SSSE3-SLOW-NEXT:    movaps %xmm8, %xmm1
; SSSE3-SLOW-NEXT:    retq
;
; SSSE3-FAST-LABEL: pair_sum_v8i32_v4i32:
; SSSE3-FAST:       # %bb.0:
; SSSE3-FAST-NEXT:    phaddd %xmm1, %xmm0
; SSSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
; SSSE3-FAST-NEXT:    phaddd %xmm3, %xmm2
; SSSE3-FAST-NEXT:    movdqa %xmm5, %xmm1
; SSSE3-FAST-NEXT:    phaddd %xmm4, %xmm5
; SSSE3-FAST-NEXT:    phaddd %xmm4, %xmm4
; SSSE3-FAST-NEXT:    phaddd %xmm1, %xmm1
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm4[1,1]
; SSSE3-FAST-NEXT:    movdqa %xmm2, %xmm3
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,2],xmm5[2,0]
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[2,0]
; SSSE3-FAST-NEXT:    paddd %xmm3, %xmm2
; SSSE3-FAST-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; SSSE3-FAST-NEXT:    phaddd %xmm6, %xmm6
; SSSE3-FAST-NEXT:    phaddd %xmm7, %xmm7
; SSSE3-FAST-NEXT:    phaddd %xmm7, %xmm6
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[2,3],xmm6[0,2]
; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm1
; SSSE3-FAST-NEXT:    retq
;
; AVX1-SLOW-LABEL: pair_sum_v8i32_v4i32:
; AVX1-SLOW:       # %bb.0:
; AVX1-SLOW-NEXT:    vphaddd %xmm0, %xmm0, %xmm8
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm8[1,1,1,1]
; AVX1-SLOW-NEXT:    vpaddd %xmm0, %xmm8, %xmm8
; AVX1-SLOW-NEXT:    vphaddd %xmm1, %xmm1, %xmm1
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; AVX1-SLOW-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
; AVX1-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
; AVX1-SLOW-NEXT:    vphaddd %xmm3, %xmm2, %xmm1
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
; AVX1-SLOW-NEXT:    vphaddd %xmm2, %xmm2, %xmm2
; AVX1-SLOW-NEXT:    vphaddd %xmm3, %xmm3, %xmm3
; AVX1-SLOW-NEXT:    vphaddd %xmm4, %xmm4, %xmm4
; AVX1-SLOW-NEXT:    vphaddd %xmm5, %xmm5, %xmm5
; AVX1-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm5[0,0,0,0]
; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7]
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7]
; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm4[1],zero
; AVX1-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[1]
; AVX1-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm8[0],xmm0[0]
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; AVX1-SLOW-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
; AVX1-SLOW-NEXT:    vphaddd %xmm6, %xmm6, %xmm2
; AVX1-SLOW-NEXT:    vphaddd %xmm7, %xmm7, %xmm3
; AVX1-SLOW-NEXT:    vphaddd %xmm3, %xmm2, %xmm2
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
; AVX1-SLOW-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
; AVX1-SLOW-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2]
; AVX1-SLOW-NEXT:    retq
;
; AVX1-FAST-LABEL: pair_sum_v8i32_v4i32:
; AVX1-FAST:       # %bb.0:
; AVX1-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm8
; AVX1-FAST-NEXT:    vphaddd %xmm3, %xmm2, %xmm1
; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
; AVX1-FAST-NEXT:    vphaddd %xmm2, %xmm2, %xmm2
; AVX1-FAST-NEXT:    vphaddd %xmm3, %xmm3, %xmm3
; AVX1-FAST-NEXT:    vphaddd %xmm4, %xmm4, %xmm4
; AVX1-FAST-NEXT:    vphaddd %xmm5, %xmm5, %xmm5
; AVX1-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm5[0,0,0,0]
; AVX1-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7]
; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
; AVX1-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7]
; AVX1-FAST-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm4[1],zero
; AVX1-FAST-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[3]
; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm8[0],xmm0[0]
; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
; AVX1-FAST-NEXT:    vphaddd %xmm7, %xmm6, %xmm2
; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm2, %xmm2
; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
; AVX1-FAST-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2]
; AVX1-FAST-NEXT:    retq
;
; AVX2-SLOW-LABEL: pair_sum_v8i32_v4i32:
; AVX2-SLOW:       # %bb.0:
; AVX2-SLOW-NEXT:    vphaddd %xmm0, %xmm0, %xmm8
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm8[1,1,1,1]
; AVX2-SLOW-NEXT:    vpaddd %xmm0, %xmm8, %xmm8
; AVX2-SLOW-NEXT:    vphaddd %xmm1, %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; AVX2-SLOW-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
; AVX2-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
; AVX2-SLOW-NEXT:    vphaddd %xmm4, %xmm4, %xmm1
; AVX2-SLOW-NEXT:    vphaddd %xmm5, %xmm5, %xmm4
; AVX2-SLOW-NEXT:    vphaddd %xmm3, %xmm2, %xmm2
; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3]
; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
; AVX2-SLOW-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1]
; AVX2-SLOW-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
; AVX2-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX2-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX2-SLOW-NEXT:    vphaddd %xmm6, %xmm6, %xmm1
; AVX2-SLOW-NEXT:    vphaddd %xmm7, %xmm7, %xmm2
; AVX2-SLOW-NEXT:    vphaddd %xmm2, %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
; AVX2-SLOW-NEXT:    vpbroadcastq %xmm1, %ymm1
; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
; AVX2-SLOW-NEXT:    retq
;
; AVX2-FAST-LABEL: pair_sum_v8i32_v4i32:
; AVX2-FAST:       # %bb.0:
; AVX2-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
; AVX2-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm1
; AVX2-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
; AVX2-FAST-NEXT:    vphaddd %xmm4, %xmm4, %xmm4
; AVX2-FAST-NEXT:    vphaddd %xmm5, %xmm5, %xmm5
; AVX2-FAST-NEXT:    vphaddd %xmm3, %xmm2, %xmm2
; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm4[0,3]
; AVX2-FAST-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm5[0]
; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[1,3],xmm4[1,3]
; AVX2-FAST-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0,1,2],xmm5[3]
; AVX2-FAST-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
; AVX2-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
; AVX2-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
; AVX2-FAST-NEXT:    vphaddd %xmm7, %xmm6, %xmm2
; AVX2-FAST-NEXT:    vphaddd %xmm1, %xmm2, %xmm1
; AVX2-FAST-NEXT:    vpbroadcastq %xmm1, %ymm1
; AVX2-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
; AVX2-FAST-NEXT:    retq
  %9 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
  %10 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
  %11 = add <2 x i32> %9, %10
  %12 = shufflevector <2 x i32> %11, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
  %13 = add <2 x i32> %11, %12
  %14 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
  %15 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
  %16 = add <2 x i32> %14, %15
  %17 = shufflevector <2 x i32> %16, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
  %18 = add <2 x i32> %16, %17
  %19 = shufflevector <2 x i32> %13, <2 x i32> %18, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %20 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
  %21 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
  %22 = add <2 x i32> %20, %21
  %23 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
  %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
  %25 = add <2 x i32> %23, %24
  %26 = shufflevector <4 x i32> %4, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
  %27 = shufflevector <4 x i32> %4, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
  %28 = add <2 x i32> %26, %27
  %29 = shufflevector <2 x i32> %28, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  %30 = shufflevector <4 x i32> %5, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
  %31 = shufflevector <4 x i32> %5, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
  %32 = add <2 x i32> %30, %31
  %33 = shufflevector <2 x i32> %32, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  %34 = shufflevector <2 x i32> %22, <2 x i32> %25, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
  %35 = shufflevector <4 x i32> %34, <4 x i32> %29, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
  %36 = shufflevector <4 x i32> %35, <4 x i32> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
  %37 = shufflevector <2 x i32> %22, <2 x i32> %25, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
  %38 = shufflevector <4 x i32> %37, <4 x i32> %29, <4 x i32> <i32 0, i32 1, i32 5, i32 undef>
  %39 = shufflevector <4 x i32> %38, <4 x i32> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 5>
  %40 = add <4 x i32> %36, %39
  %41 = shufflevector <4 x i32> %40, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
  %42 = shufflevector <8 x i32> %19, <8 x i32> %41, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef>
  %43 = shufflevector <4 x i32> %6, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
  %44 = shufflevector <4 x i32> %6, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
  %45 = add <2 x i32> %43, %44
  %46 = shufflevector <4 x i32> %7, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
  %47 = shufflevector <4 x i32> %7, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
  %48 = add <2 x i32> %46, %47
  %49 = shufflevector <2 x i32> %45, <2 x i32> %48, <2 x i32> <i32 0, i32 2>
  %50 = shufflevector <2 x i32> %45, <2 x i32> %48, <2 x i32> <i32 1, i32 3>
  %51 = add <2 x i32> %49, %50
  %52 = shufflevector <2 x i32> %51, <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  %53 = shufflevector <8 x i32> %42, <8 x i32> %52, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
  ret <8 x i32> %53
}

; Vectorized Sequential Sum Reductions
; e.g.
; inline STYPE sum(VTYPE x) {
;   return ((x[0] + x[1]) + x[2]) + x[3];
; }
;
; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) {
;   return (VTYPE) { sum( A0 ), sum( A1 ), sum( A2 ), sum( A3 ) };
; }

define <4 x float> @sequential_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {
; SSSE3-SLOW-LABEL: sequential_sum_v4f32_v4f32:
; SSSE3-SLOW:       # %bb.0:
; SSSE3-SLOW-NEXT:    movaps %xmm0, %xmm4
; SSSE3-SLOW-NEXT:    movaps %xmm0, %xmm5
; SSSE3-SLOW-NEXT:    haddps %xmm1, %xmm5
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,2,2,3]
; SSSE3-SLOW-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; SSSE3-SLOW-NEXT:    addps %xmm5, %xmm0
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3],xmm4[3,3]
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0,2,3]
; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm0
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm2, %xmm1
; SSSE3-SLOW-NEXT:    movaps %xmm2, %xmm4
; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1]
; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm4
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm4, %xmm2
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm3, %xmm1
; SSSE3-SLOW-NEXT:    movaps %xmm3, %xmm4
; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm4
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm4, %xmm3
; SSSE3-SLOW-NEXT:    movlhps {{.*#+}} xmm3 = xmm3[0],xmm2[0]
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
; SSSE3-SLOW-NEXT:    retq
;
; SSSE3-FAST-LABEL: sequential_sum_v4f32_v4f32:
; SSSE3-FAST:       # %bb.0:
; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm4
; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm5
; SSSE3-FAST-NEXT:    haddps %xmm1, %xmm5
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,2,2,3]
; SSSE3-FAST-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; SSSE3-FAST-NEXT:    addps %xmm5, %xmm0
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3],xmm4[3,3]
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0,2,3]
; SSSE3-FAST-NEXT:    addps %xmm1, %xmm0
; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm1
; SSSE3-FAST-NEXT:    haddps %xmm2, %xmm1
; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm4
; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1]
; SSSE3-FAST-NEXT:    addps %xmm1, %xmm4
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
; SSSE3-FAST-NEXT:    addps %xmm4, %xmm2
; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm1
; SSSE3-FAST-NEXT:    haddps %xmm3, %xmm1
; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm4
; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
; SSSE3-FAST-NEXT:    addps %xmm1, %xmm4
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
; SSSE3-FAST-NEXT:    addps %xmm4, %xmm3
; SSSE3-FAST-NEXT:    movlhps {{.*#+}} xmm3 = xmm3[0],xmm2[0]
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
; SSSE3-FAST-NEXT:    retq
;
; AVX-SLOW-LABEL: sequential_sum_v4f32_v4f32:
; AVX-SLOW:       # %bb.0:
; AVX-SLOW-NEXT:    vhaddps %xmm1, %xmm0, %xmm4
; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm4 = xmm4[0,2,2,3]
; AVX-SLOW-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX-SLOW-NEXT:    vaddps %xmm4, %xmm5, %xmm4
; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
; AVX-SLOW-NEXT:    vaddps %xmm4, %xmm0, %xmm0
; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
; AVX-SLOW-NEXT:    vaddps %xmm2, %xmm1, %xmm1
; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm4, %xmm1
; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm2, %xmm1
; AVX-SLOW-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
; AVX-SLOW-NEXT:    vaddps %xmm3, %xmm1, %xmm1
; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm2, %xmm1
; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3]
; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm2, %xmm1
; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
; AVX-SLOW-NEXT:    retq
;
; AVX-FAST-LABEL: sequential_sum_v4f32_v4f32:
; AVX-FAST:       # %bb.0:
; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm4
; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm4 = xmm4[0,2,2,3]
; AVX-FAST-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX-FAST-NEXT:    vaddps %xmm4, %xmm5, %xmm4
; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
; AVX-FAST-NEXT:    vaddps %xmm4, %xmm0, %xmm0
; AVX-FAST-NEXT:    vhaddps %xmm2, %xmm2, %xmm1
; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
; AVX-FAST-NEXT:    vaddps %xmm1, %xmm4, %xmm1
; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
; AVX-FAST-NEXT:    vaddps %xmm1, %xmm2, %xmm1
; AVX-FAST-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX-FAST-NEXT:    vhaddps %xmm3, %xmm3, %xmm1
; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
; AVX-FAST-NEXT:    vaddps %xmm1, %xmm2, %xmm1
; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3]
; AVX-FAST-NEXT:    vaddps %xmm1, %xmm2, %xmm1
; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
; AVX-FAST-NEXT:    retq
  %5 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 0, i32 4>
  %6 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 1, i32 5>
  %7 = fadd <2 x float> %5, %6
  %8 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 2, i32 6>
  %9 = fadd <2 x float> %8, %7
  %10 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 3, i32 7>
  %11 = fadd <2 x float> %10, %9
  %12 = shufflevector <2 x float> %11, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  %13 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
  %14 = fadd <4 x float> %13, %2
  %15 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
  %16 = fadd <4 x float> %15, %14
  %17 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
  %18 = fadd <4 x float> %17, %16
  %19 = shufflevector <4 x float> %12, <4 x float> %18, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
  %20 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
  %21 = fadd <4 x float> %20, %3
  %22 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
  %23 = fadd <4 x float> %22, %21
  %24 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
  %25 = fadd <4 x float> %24, %23
  %26 = shufflevector <4 x float> %19, <4 x float> %25, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
  ret <4 x float> %26
}

define <4 x i32> @sequential_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) {
; SSSE3-SLOW-LABEL: sequential_sum_v4i32_v4i32:
; SSSE3-SLOW:       # %bb.0:
; SSSE3-SLOW-NEXT:    movdqa %xmm0, %xmm4
; SSSE3-SLOW-NEXT:    phaddd %xmm1, %xmm4
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
; SSSE3-SLOW-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSSE3-SLOW-NEXT:    paddd %xmm4, %xmm1
; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
; SSSE3-SLOW-NEXT:    paddd %xmm2, %xmm4
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
; SSSE3-SLOW-NEXT:    paddd %xmm4, %xmm2
; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm2
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
; SSSE3-SLOW-NEXT:    paddd %xmm3, %xmm4
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
; SSSE3-SLOW-NEXT:    paddd %xmm4, %xmm3
; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm3
; SSSE3-SLOW-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
; SSSE3-SLOW-NEXT:    retq
;
; SSSE3-FAST-LABEL: sequential_sum_v4i32_v4i32:
; SSSE3-FAST:       # %bb.0:
; SSSE3-FAST-NEXT:    movdqa %xmm0, %xmm4
; SSSE3-FAST-NEXT:    phaddd %xmm1, %xmm4
; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
; SSSE3-FAST-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSSE3-FAST-NEXT:    paddd %xmm4, %xmm1
; SSSE3-FAST-NEXT:    paddd %xmm1, %xmm0
; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]
; SSSE3-FAST-NEXT:    phaddd %xmm2, %xmm2
; SSSE3-FAST-NEXT:    paddd %xmm2, %xmm4
; SSSE3-FAST-NEXT:    paddd %xmm1, %xmm4
; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[3,3,3,3]
; SSSE3-FAST-NEXT:    phaddd %xmm3, %xmm3
; SSSE3-FAST-NEXT:    paddd %xmm3, %xmm2
; SSSE3-FAST-NEXT:    paddd %xmm1, %xmm2
; SSSE3-FAST-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
; SSSE3-FAST-NEXT:    retq
;
; AVX1-SLOW-LABEL: sequential_sum_v4i32_v4i32:
; AVX1-SLOW:       # %bb.0:
; AVX1-SLOW-NEXT:    vphaddd %xmm1, %xmm0, %xmm4
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
; AVX1-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
; AVX1-SLOW-NEXT:    vpaddd %xmm0, %xmm4, %xmm0
; AVX1-SLOW-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
; AVX1-SLOW-NEXT:    vpaddd %xmm4, %xmm2, %xmm4
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm4, %xmm2
; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
; AVX1-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
; AVX1-SLOW-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
; AVX1-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
; AVX1-SLOW-NEXT:    retq
;
; AVX1-FAST-LABEL: sequential_sum_v4i32_v4i32:
; AVX1-FAST:       # %bb.0:
; AVX1-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm4
; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
; AVX1-FAST-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX1-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
; AVX1-FAST-NEXT:    vpaddd %xmm0, %xmm4, %xmm0
; AVX1-FAST-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
; AVX1-FAST-NEXT:    vphaddd %xmm2, %xmm2, %xmm1
; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
; AVX1-FAST-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm4, %xmm1
; AVX1-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX1-FAST-NEXT:    vphaddd %xmm3, %xmm3, %xmm1
; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
; AVX1-FAST-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
; AVX1-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
; AVX1-FAST-NEXT:    retq
;
; AVX2-SLOW-LABEL: sequential_sum_v4i32_v4i32:
; AVX2-SLOW:       # %bb.0:
; AVX2-SLOW-NEXT:    vphaddd %xmm1, %xmm0, %xmm4
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
; AVX2-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
; AVX2-SLOW-NEXT:    vpaddd %xmm0, %xmm4, %xmm0
; AVX2-SLOW-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
; AVX2-SLOW-NEXT:    vpaddd %xmm4, %xmm2, %xmm4
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
; AVX2-SLOW-NEXT:    vpaddd %xmm2, %xmm4, %xmm2
; AVX2-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
; AVX2-SLOW-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
; AVX2-SLOW-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
; AVX2-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vpbroadcastd %xmm1, %xmm1
; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
; AVX2-SLOW-NEXT:    retq
;
; AVX2-FAST-LABEL: sequential_sum_v4i32_v4i32:
; AVX2-FAST:       # %bb.0:
; AVX2-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm4
; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
; AVX2-FAST-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX2-FAST-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
; AVX2-FAST-NEXT:    vpaddd %xmm0, %xmm4, %xmm0
; AVX2-FAST-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
; AVX2-FAST-NEXT:    vphaddd %xmm2, %xmm2, %xmm1
; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
; AVX2-FAST-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
; AVX2-FAST-NEXT:    vpaddd %xmm1, %xmm4, %xmm1
; AVX2-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX2-FAST-NEXT:    vphaddd %xmm3, %xmm3, %xmm1
; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
; AVX2-FAST-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
; AVX2-FAST-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
; AVX2-FAST-NEXT:    vpbroadcastd %xmm1, %xmm1
; AVX2-FAST-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
; AVX2-FAST-NEXT:    retq
  %5 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 0, i32 4>
  %6 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 1, i32 5>
  %7 = add <2 x i32> %5, %6
  %8 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 2, i32 6>
  %9 = add <2 x i32> %8, %7
  %10 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 3, i32 7>
  %11 = add <2 x i32> %10, %9
  %12 = shufflevector <2 x i32> %11, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  %13 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
  %14 = add <4 x i32> %13, %2
  %15 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
  %16 = add <4 x i32> %15, %14
  %17 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
  %18 = add <4 x i32> %17, %16
  %19 = shufflevector <4 x i32> %12, <4 x i32> %18, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
  %20 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
  %21 = add <4 x i32> %20, %3
  %22 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
  %23 = add <4 x i32> %22, %21
  %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
  %25 = add <4 x i32> %24, %23
  %26 = shufflevector <4 x i32> %19, <4 x i32> %25, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
  ret <4 x i32> %26
}

; Vectorized Reductions
; e.g.
; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) {
;   return (VTYPE) { reduce( A0 ), reduce( A1 ), reduce( A2 ), reduce( A3 ) };
; }

define <4 x float> @reduction_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {
; SSSE3-SLOW-LABEL: reduction_sum_v4f32_v4f32:
; SSSE3-SLOW:       # %bb.0:
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
; SSSE3-SLOW-NEXT:    addss %xmm0, %xmm4
; SSSE3-SLOW-NEXT:    movaps %xmm0, %xmm5
; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1]
; SSSE3-SLOW-NEXT:    addss %xmm4, %xmm5
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
; SSSE3-SLOW-NEXT:    addss %xmm5, %xmm0
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
; SSSE3-SLOW-NEXT:    addss %xmm1, %xmm4
; SSSE3-SLOW-NEXT:    movaps %xmm1, %xmm5
; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]
; SSSE3-SLOW-NEXT:    addss %xmm4, %xmm5
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
; SSSE3-SLOW-NEXT:    addss %xmm5, %xmm1
; SSSE3-SLOW-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
; SSSE3-SLOW-NEXT:    addss %xmm2, %xmm1
; SSSE3-SLOW-NEXT:    movaps %xmm2, %xmm4
; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1]
; SSSE3-SLOW-NEXT:    addss %xmm1, %xmm4
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
; SSSE3-SLOW-NEXT:    addss %xmm4, %xmm2
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
; SSSE3-SLOW-NEXT:    addss %xmm3, %xmm1
; SSSE3-SLOW-NEXT:    movaps %xmm3, %xmm4
; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
; SSSE3-SLOW-NEXT:    addss %xmm1, %xmm4
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
; SSSE3-SLOW-NEXT:    addss %xmm4, %xmm3
; SSSE3-SLOW-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; SSSE3-SLOW-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; SSSE3-SLOW-NEXT:    retq
;
; SSSE3-FAST-LABEL: reduction_sum_v4f32_v4f32:
; SSSE3-FAST:       # %bb.0:
; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm4
; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm4
; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm5
; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1]
; SSSE3-FAST-NEXT:    addss %xmm4, %xmm5
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
; SSSE3-FAST-NEXT:    addss %xmm5, %xmm0
; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm4
; SSSE3-FAST-NEXT:    haddps %xmm1, %xmm4
; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm5
; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]
; SSSE3-FAST-NEXT:    addss %xmm4, %xmm5
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
; SSSE3-FAST-NEXT:    addss %xmm5, %xmm1
; SSSE3-FAST-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm1
; SSSE3-FAST-NEXT:    haddps %xmm2, %xmm1
; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm4
; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1]
; SSSE3-FAST-NEXT:    addss %xmm1, %xmm4
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
; SSSE3-FAST-NEXT:    addss %xmm4, %xmm2
; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm1
; SSSE3-FAST-NEXT:    haddps %xmm3, %xmm1
; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm4
; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
; SSSE3-FAST-NEXT:    addss %xmm1, %xmm4
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
; SSSE3-FAST-NEXT:    addss %xmm4, %xmm3
; SSSE3-FAST-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; SSSE3-FAST-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; SSSE3-FAST-NEXT:    retq
;
; AVX-SLOW-LABEL: reduction_sum_v4f32_v4f32:
; AVX-SLOW:       # %bb.0:
; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
; AVX-SLOW-NEXT:    vaddss %xmm4, %xmm0, %xmm4
; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm5 = xmm0[1,0]
; AVX-SLOW-NEXT:    vaddss %xmm5, %xmm4, %xmm4
; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX-SLOW-NEXT:    vaddss %xmm0, %xmm4, %xmm0
; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
; AVX-SLOW-NEXT:    vaddss %xmm4, %xmm1, %xmm4
; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm5 = xmm1[1,0]
; AVX-SLOW-NEXT:    vaddss %xmm5, %xmm4, %xmm4
; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm4, %xmm1
; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm2, %xmm1
; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
; AVX-SLOW-NEXT:    vaddss %xmm4, %xmm1, %xmm1
; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
; AVX-SLOW-NEXT:    vaddss %xmm2, %xmm1, %xmm1
; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm3, %xmm1
; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
; AVX-SLOW-NEXT:    vaddss %xmm2, %xmm1, %xmm1
; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3]
; AVX-SLOW-NEXT:    vaddss %xmm2, %xmm1, %xmm1
; AVX-SLOW-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
; AVX-SLOW-NEXT:    retq
;
; AVX-FAST-LABEL: reduction_sum_v4f32_v4f32:
; AVX-FAST:       # %bb.0:
; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm4
; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm5 = xmm0[1,0]
; AVX-FAST-NEXT:    vaddss %xmm5, %xmm4, %xmm4
; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX-FAST-NEXT:    vaddss %xmm0, %xmm4, %xmm0
; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm4
; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm5 = xmm1[1,0]
; AVX-FAST-NEXT:    vaddss %xmm5, %xmm4, %xmm4
; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
; AVX-FAST-NEXT:    vaddss %xmm1, %xmm4, %xmm1
; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
; AVX-FAST-NEXT:    vhaddps %xmm2, %xmm2, %xmm1
; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
; AVX-FAST-NEXT:    vaddss %xmm4, %xmm1, %xmm1
; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
; AVX-FAST-NEXT:    vaddss %xmm2, %xmm1, %xmm1
; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
; AVX-FAST-NEXT:    vhaddps %xmm3, %xmm3, %xmm1
; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
; AVX-FAST-NEXT:    vaddss %xmm2, %xmm1, %xmm1
; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3]
; AVX-FAST-NEXT:    vaddss %xmm2, %xmm1, %xmm1
; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
; AVX-FAST-NEXT:    retq
  %5 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %0)
  %6 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %1)
  %7 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %2)
  %8 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %3)
  %9 = insertelement <4 x float> undef, float %5, i32 0
  %10 = insertelement <4 x float> %9,   float %6, i32 1
  %11 = insertelement <4 x float> %10,  float %7, i32 2
  %12 = insertelement <4 x float> %11,  float %8, i32 3
  ret <4 x float> %12
}
declare float @llvm.vector.reduce.fadd.f32.v4f32(float, <4 x float>)

define <4 x float> @reduction_sum_v4f32_v4f32_reassoc(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {
; SSSE3-SLOW-LABEL: reduction_sum_v4f32_v4f32_reassoc:
; SSSE3-SLOW:       # %bb.0:
; SSSE3-SLOW-NEXT:    movaps %xmm0, %xmm4
; SSSE3-SLOW-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
; SSSE3-SLOW-NEXT:    addps %xmm4, %xmm0
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm4, %xmm0
; SSSE3-SLOW-NEXT:    movaps %xmm1, %xmm4
; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm1[1]
; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm4
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm4[1,1,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm4, %xmm1
; SSSE3-SLOW-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; SSSE3-SLOW-NEXT:    movaps %xmm2, %xmm1
; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
; SSSE3-SLOW-NEXT:    addps %xmm2, %xmm1
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm2
; SSSE3-SLOW-NEXT:    movaps %xmm3, %xmm1
; SSSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
; SSSE3-SLOW-NEXT:    addps %xmm3, %xmm1
; SSSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
; SSSE3-SLOW-NEXT:    addps %xmm1, %xmm3
; SSSE3-SLOW-NEXT:    movlhps {{.*#+}} xmm3 = xmm3[0],xmm2[0]
; SSSE3-SLOW-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
; SSSE3-SLOW-NEXT:    retq
;
; SSSE3-FAST-LABEL: reduction_sum_v4f32_v4f32_reassoc:
; SSSE3-FAST:       # %bb.0:
; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm4
; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
; SSSE3-FAST-NEXT:    addps %xmm0, %xmm4
; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm0
; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
; SSSE3-FAST-NEXT:    addps %xmm1, %xmm0
; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm4
; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm0
; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]
; SSSE3-FAST-NEXT:    addps %xmm2, %xmm0
; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm1
; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
; SSSE3-FAST-NEXT:    addps %xmm3, %xmm1
; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm1
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,2],xmm1[2,0]
; SSSE3-FAST-NEXT:    movaps %xmm4, %xmm0
; SSSE3-FAST-NEXT:    retq
;
; AVX-SLOW-LABEL: reduction_sum_v4f32_v4f32_reassoc:
; AVX-SLOW:       # %bb.0:
; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm0[1,0]
; AVX-SLOW-NEXT:    vaddps %xmm4, %xmm0, %xmm0
; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
; AVX-SLOW-NEXT:    vaddps %xmm4, %xmm0, %xmm0
; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm1[1,0]
; AVX-SLOW-NEXT:    vaddps %xmm4, %xmm1, %xmm1
; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
; AVX-SLOW-NEXT:    vaddps %xmm4, %xmm1, %xmm1
; AVX-SLOW-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm2[1,0]
; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm2, %xmm1
; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
; AVX-SLOW-NEXT:    vaddps %xmm2, %xmm1, %xmm1
; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
; AVX-SLOW-NEXT:    vaddps %xmm2, %xmm3, %xmm2
; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm2[1,1,3,3]
; AVX-SLOW-NEXT:    vaddps %xmm3, %xmm2, %xmm2
; AVX-SLOW-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,0]
; AVX-SLOW-NEXT:    retq
;
; AVX-FAST-LABEL: reduction_sum_v4f32_v4f32_reassoc:
; AVX-FAST:       # %bb.0:
; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm0[1,0]
; AVX-FAST-NEXT:    vaddps %xmm4, %xmm0, %xmm0
; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm4 = xmm1[1,0]
; AVX-FAST-NEXT:    vaddps %xmm4, %xmm1, %xmm1
; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm2[1,0]
; AVX-FAST-NEXT:    vaddps %xmm1, %xmm2, %xmm1
; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
; AVX-FAST-NEXT:    vaddps %xmm2, %xmm3, %xmm2
; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm2, %xmm1
; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[2,0]
; AVX-FAST-NEXT:    retq
  %5 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %0)
  %6 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %1)
  %7 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %2)
  %8 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %3)
  %9 = insertelement <4 x float> undef, float %5, i32 0
  %10 = insertelement <4 x float> %9,   float %6, i32 1
  %11 = insertelement <4 x float> %10,  float %7, i32 2
  %12 = insertelement <4 x float> %11,  float %8, i32 3
  ret <4 x float> %12
}

define <4 x i32> @reduction_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) {
; SSSE3-SLOW-LABEL: reduction_sum_v4i32_v4i32:
; SSSE3-SLOW:       # %bb.0:
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
; SSSE3-SLOW-NEXT:    paddd %xmm0, %xmm4
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,1,1]
; SSSE3-SLOW-NEXT:    paddd %xmm4, %xmm0
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm4
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,1,1]
; SSSE3-SLOW-NEXT:    paddd %xmm4, %xmm1
; SSSE3-SLOW-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
; SSSE3-SLOW-NEXT:    paddd %xmm2, %xmm1
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm2
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
; SSSE3-SLOW-NEXT:    paddd %xmm3, %xmm1
; SSSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
; SSSE3-SLOW-NEXT:    paddd %xmm1, %xmm3
; SSSE3-SLOW-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
; SSSE3-SLOW-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; SSSE3-SLOW-NEXT:    retq
;
; SSSE3-FAST-LABEL: reduction_sum_v4i32_v4i32:
; SSSE3-FAST:       # %bb.0:
; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
; SSSE3-FAST-NEXT:    paddd %xmm4, %xmm0
; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
; SSSE3-FAST-NEXT:    paddd %xmm1, %xmm4
; SSSE3-FAST-NEXT:    phaddd %xmm4, %xmm0
; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
; SSSE3-FAST-NEXT:    paddd %xmm2, %xmm1
; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
; SSSE3-FAST-NEXT:    paddd %xmm3, %xmm2
; SSSE3-FAST-NEXT:    phaddd %xmm2, %xmm1
; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
; SSSE3-FAST-NEXT:    retq
;
; AVX-SLOW-LABEL: reduction_sum_v4i32_v4i32:
; AVX-SLOW:       # %bb.0:
; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
; AVX-SLOW-NEXT:    vpaddd %xmm4, %xmm0, %xmm0
; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
; AVX-SLOW-NEXT:    vpaddd %xmm4, %xmm0, %xmm0
; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
; AVX-SLOW-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
; AVX-SLOW-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
; AVX-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
; AVX-SLOW-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
; AVX-SLOW-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,1,1]
; AVX-SLOW-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
; AVX-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
; AVX-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX-SLOW-NEXT:    retq
;
; AVX-FAST-LABEL: reduction_sum_v4i32_v4i32:
; AVX-FAST:       # %bb.0:
; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
; AVX-FAST-NEXT:    vpaddd %xmm4, %xmm0, %xmm0
; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
; AVX-FAST-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
; AVX-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
; AVX-FAST-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
; AVX-FAST-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
; AVX-FAST-NEXT:    vphaddd %xmm2, %xmm1, %xmm1
; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
; AVX-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX-FAST-NEXT:    retq
  %5 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %0)
  %6 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %1)
  %7 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %2)
  %8 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %3)
  %9 = insertelement <4 x i32> undef, i32 %5, i32 0
  %10 = insertelement <4 x i32> %9,   i32 %6, i32 1
  %11 = insertelement <4 x i32> %10,  i32 %7, i32 2
  %12 = insertelement <4 x i32> %11,  i32 %8, i32 3
  ret <4 x i32> %12
}
declare i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32>)