test/CodeGen/X86/combine-fma-concat.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver2 | FileCheck %s --check-prefixes=FMA4
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=FMA3,AVX2
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=FMA3,AVX512

; 2 constants - only single operand to concat
define <4 x double> @concat_fmadd_v4f64_v2f64_constants(<2 x double> %a0, <2 x double> %a1) {
; FMA4-LABEL: concat_fmadd_v4f64_v2f64_constants:
; FMA4:       # %bb.0:
; FMA4-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; FMA4-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; FMA4-NEXT:    vmovapd {{.*#+}} ymm1 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
; FMA4-NEXT:    vfmaddpd {{.*#+}} ymm0 = (ymm0 * ymm1) + mem
; FMA4-NEXT:    retq
;
; AVX2-LABEL: concat_fmadd_v4f64_v2f64_constants:
; AVX2:       # %bb.0:
; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
; AVX2-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2
; AVX2-NEXT:    retq
;
; AVX512-LABEL: concat_fmadd_v4f64_v2f64_constants:
; AVX512:       # %bb.0:
; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
; AVX512-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
; AVX512-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + mem
; AVX512-NEXT:    retq
  %v0 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a0, <2 x double> splat (double 2.0), <2 x double> splat (double 1.0))
  %v1 = call <2 x double> @llvm.fma.v2f64(<2 x double> %a1, <2 x double> splat (double 2.0), <2 x double> splat (double 1.0))
  %res = shufflevector <2 x double> %v0, <2 x double> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  ret <4 x double> %res
}

; 1 sitofp + 1 constant - only single operand to concat
define <8 x float> @concat_fmadd_v8f32_v4f32_constant_sitofp(<4 x float> %a0, <4 x float> %a1, <4 x i32> %b0, <4 x i32> %b1) {
; FMA4-LABEL: concat_fmadd_v8f32_v4f32_constant_sitofp:
; FMA4:       # %bb.0:
; FMA4-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; FMA4-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
; FMA4-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; FMA4-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm1
; FMA4-NEXT:    vcvtdq2ps %ymm1, %ymm1
; FMA4-NEXT:    vfmaddps {{.*#+}} ymm0 = (ymm0 * ymm1) + mem
; FMA4-NEXT:    retq
;
; AVX2-LABEL: concat_fmadd_v8f32_v4f32_constant_sitofp:
; AVX2:       # %bb.0:
; AVX2-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
; AVX2-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm0
; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm2
; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm0 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
; AVX2-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0
; AVX2-NEXT:    retq
;
; AVX512-LABEL: concat_fmadd_v8f32_v4f32_constant_sitofp:
; AVX512:       # %bb.0:
; AVX512-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
; AVX512-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm0
; AVX512-NEXT:    vcvtdq2ps %ymm0, %ymm0
; AVX512-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + mem
; AVX512-NEXT:    retq
  %i0 = sitofp <4 x i32> %b0 to <4 x float>
  %i1 = sitofp <4 x i32> %b1 to <4 x float>
  %v0 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %i0, <4 x float> splat (float 1.0))
  %v1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a1, <4 x float> %i1, <4 x float> splat (float 1.0))
  %res = shufflevector <4 x float> %v0, <4 x float> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  ret <8 x float> %res
}

; 1 fneg (fnmadd) + 2 constant - only single operand to concat
define <8 x double> @concat_fnmadd_v8f64_v2f64_constants(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> %a3) {
; FMA4-LABEL: concat_fnmadd_v8f64_v2f64_constants:
; FMA4:       # %bb.0:
; FMA4-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; FMA4-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; FMA4-NEXT:    vmovapd {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
; FMA4-NEXT:    vmovapd {{.*#+}} ymm4 = [4.0E+0,4.0E+0,4.0E+0,4.0E+0]
; FMA4-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
; FMA4-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
; FMA4-NEXT:    vfmaddpd {{.*#+}} ymm0 = (ymm0 * ymm4) + ymm1
; FMA4-NEXT:    vfmaddpd {{.*#+}} ymm1 = (ymm2 * ymm4) + ymm1
; FMA4-NEXT:    retq
;
; AVX2-LABEL: concat_fnmadd_v8f64_v2f64_constants:
; AVX2:       # %bb.0:
; AVX2-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [4.0E+0,4.0E+0,4.0E+0,4.0E+0]
; AVX2-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm5 * ymm0) + ymm4
; AVX2-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm1
; AVX2-NEXT:    vfmadd213pd {{.*#+}} ymm1 = (ymm5 * ymm1) + ymm4
; AVX2-NEXT:    retq
;
; AVX512-LABEL: concat_fnmadd_v8f64_v2f64_constants:
; AVX512:       # %bb.0:
; AVX512-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX512-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm1
; AVX512-NEXT:    vbroadcastsd {{.*#+}} zmm0 = [-4.0E+0,-4.0E+0,-4.0E+0,-4.0E+0,-4.0E+0,-4.0E+0,-4.0E+0,-4.0E+0]
; AVX512-NEXT:    vfnmadd213pd {{.*#+}} zmm0 = -(zmm1 * zmm0) + mem
; AVX512-NEXT:    retq
  %n0 = fneg <2 x double> %a0
  %n1 = fneg <2 x double> %a1
  %n2 = fneg <2 x double> %a2
  %n3 = fneg <2 x double> %a3
  %v0 = call <2 x double> @llvm.fma.v2f64(<2 x double> %n0, <2 x double> splat (double -4.0), <2 x double> splat (double 1.0))
  %v1 = call <2 x double> @llvm.fma.v2f64(<2 x double> %n1, <2 x double> splat (double -4.0), <2 x double> splat (double 1.0))
  %v2 = call <2 x double> @llvm.fma.v2f64(<2 x double> %n2, <2 x double> splat (double -4.0), <2 x double> splat (double 1.0))
  %v3 = call <2 x double> @llvm.fma.v2f64(<2 x double> %n3, <2 x double> splat (double -4.0), <2 x double> splat (double 1.0))
  %r01 = shufflevector <2 x double> %v0, <2 x double> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %r23 = shufflevector <2 x double> %v2, <2 x double> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
  %res = shufflevector <4 x double> %r01, <4 x double> %r23, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  ret <8 x double> %res
}

; self mul (dot product pattern)
define <16 x float> @concat_fma_self_v16f32_v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> %a3) {
; FMA4-LABEL: concat_fma_self_v16f32_v4f32:
; FMA4:       # %bb.0:
; FMA4-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; FMA4-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; FMA4-NEXT:    vmovaps {{.*#+}} ymm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; FMA4-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
; FMA4-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
; FMA4-NEXT:    vfmaddps {{.*#+}} ymm0 = (ymm0 * ymm0) + ymm1
; FMA4-NEXT:    vfmaddps {{.*#+}} ymm1 = (ymm2 * ymm2) + ymm1
; FMA4-NEXT:    retq
;
; AVX2-LABEL: concat_fma_self_v16f32_v4f32:
; AVX2:       # %bb.0:
; AVX2-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm4 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
; AVX2-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm0 * ymm0) + ymm4
; AVX2-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm1
; AVX2-NEXT:    vfmadd213ps {{.*#+}} ymm1 = (ymm1 * ymm1) + ymm4
; AVX2-NEXT:    retq
;
; AVX512-LABEL: concat_fma_self_v16f32_v4f32:
; AVX512:       # %bb.0:
; AVX512-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX512-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512-NEXT:    vfmadd213ps {{.*#+}} zmm0 = (zmm0 * zmm0) + mem
; AVX512-NEXT:    retq
  %v0 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %a0, <4 x float> splat (float -0.0))
  %v1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a1, <4 x float> %a1, <4 x float> splat (float -0.0))
  %v2 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a2, <4 x float> %a2, <4 x float> splat (float -0.0))
  %v3 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a3, <4 x float> %a3, <4 x float> splat (float -0.0))
  %r01 = shufflevector <4 x float> %v0, <4 x float> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  %r23 = shufflevector <4 x float> %v2, <4 x float> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  %res = shufflevector <8 x float> %r01, <8 x float> %r23, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  ret <16 x float> %res
}

; 1 fneg (fmsub) + 2 constant - only single operand to concat
define <16 x float> @concat_fmsub_v16f32_v8f32_constant_split(<8 x float> %a0, <8 x float> %a1, <16 x float> %b) {
; FMA4-LABEL: concat_fmsub_v16f32_v8f32_constant_split:
; FMA4:       # %bb.0:
; FMA4-NEXT:    vmovaps {{.*#+}} ymm4 = [-8.0E+0,-8.0E+0,-8.0E+0,-8.0E+0,-8.0E+0,-8.0E+0,-8.0E+0,-8.0E+0]
; FMA4-NEXT:    vfmsubps {{.*#+}} ymm0 = (ymm0 * ymm4) - ymm2
; FMA4-NEXT:    vfmsubps {{.*#+}} ymm1 = (ymm1 * ymm4) - ymm3
; FMA4-NEXT:    retq
;
; AVX2-LABEL: concat_fmsub_v16f32_v8f32_constant_split:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm4 = [-8.0E+0,-8.0E+0,-8.0E+0,-8.0E+0,-8.0E+0,-8.0E+0,-8.0E+0,-8.0E+0]
; AVX2-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm4 * ymm0) - ymm2
; AVX2-NEXT:    vfmsub213ps {{.*#+}} ymm1 = (ymm4 * ymm1) - ymm3
; AVX2-NEXT:    retq
;
; AVX512-LABEL: concat_fmsub_v16f32_v8f32_constant_split:
; AVX512:       # %bb.0:
; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
; AVX512-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512-NEXT:    vfmsub132ps {{.*#+}} zmm0 = (zmm0 * mem) - zmm2
; AVX512-NEXT:    retq
  %b0 = shufflevector <16 x float> %b, <16 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  %b1 = shufflevector <16 x float> %b, <16 x float> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  %n0 = fneg <8 x float> %b0
  %n1 = fneg <8 x float> %b1
  %v0 = call <8 x float> @llvm.fma.v8f32(<8 x float> %a0, <8 x float> splat (float -8.0), <8 x float> %n0)
  %v1 = call <8 x float> @llvm.fma.v8f32(<8 x float> %a1, <8 x float> splat (float -8.0), <8 x float> %n1)
  %res = shufflevector <8 x float> %v0, <8 x float> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  ret <16 x float> %res
}

; 1 repeated concat + 1 constant - we only have to concat 2 operands down the fma chain
define <8 x float> @concat_fma_v8f32_v4f32_constant_repeatedop(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> %a3) {
; FMA4-LABEL: concat_fma_v8f32_v4f32_constant_repeatedop:
; FMA4:       # %bb.0:
; FMA4-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
; FMA4-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; FMA4-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
; FMA4-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; FMA4-NEXT:    vfmaddps {{.*#+}} ymm1 = (ymm0 * ymm2) + mem
; FMA4-NEXT:    vfmaddps {{.*#+}} ymm0 = (ymm0 * ymm1) + mem
; FMA4-NEXT:    retq
;
; AVX2-LABEL: concat_fma_v8f32_v4f32_constant_repeatedop:
; AVX2:       # %bb.0:
; AVX2-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
; AVX2-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm0
; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
; AVX2-NEXT:    vfmadd231ps {{.*#+}} ymm2 = (ymm1 * ymm0) + ymm2
; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
; AVX2-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0
; AVX2-NEXT:    retq
;
; AVX512-LABEL: concat_fma_v8f32_v4f32_constant_repeatedop:
; AVX512:       # %bb.0:
; AVX512-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
; AVX512-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm0
; AVX512-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + mem
; AVX512-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + mem
; AVX512-NEXT:    retq
  %l0 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %a2, <4 x float> splat (float 1.000000e+00))
  %h0 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a1, <4 x float> %a3, <4 x float> splat (float 1.000000e+00))
  %l1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %l0, <4 x float> splat (float 2.000000e+00))
  %h1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a1, <4 x float> %h0, <4 x float> splat (float 2.000000e+00))
  %r = shufflevector <4 x float> %l1, <4 x float> %h1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  ret <8 x float> %r
}

define <8 x double> @concat_fma_fmsub_v8f64_v4f64_constant_repeatedop_commute(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> %a3) {
; FMA4-LABEL: concat_fma_fmsub_v8f64_v4f64_constant_repeatedop_commute:
; FMA4:       # %bb.0:
; FMA4-NEXT:    vmovapd {{.*#+}} ymm4 = [-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0]
; FMA4-NEXT:    vfmaddpd {{.*#+}} ymm2 = (ymm2 * ymm0) + ymm4
; FMA4-NEXT:    vfmaddpd {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm4
; FMA4-NEXT:    vfmsubpd {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm4
; FMA4-NEXT:    vfmsubpd {{.*#+}} ymm1 = (ymm1 * ymm3) - ymm4
; FMA4-NEXT:    retq
;
; AVX2-LABEL: concat_fma_fmsub_v8f64_v4f64_constant_repeatedop_commute:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0]
; AVX2-NEXT:    vfmadd213pd {{.*#+}} ymm2 = (ymm0 * ymm2) + ymm4
; AVX2-NEXT:    vfmadd213pd {{.*#+}} ymm3 = (ymm1 * ymm3) + ymm4
; AVX2-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm2 * ymm0) - ymm4
; AVX2-NEXT:    vfmsub213pd {{.*#+}} ymm1 = (ymm3 * ymm1) - ymm4
; AVX2-NEXT:    retq
;
; AVX512-LABEL: concat_fma_fmsub_v8f64_v4f64_constant_repeatedop_commute:
; AVX512:       # %bb.0:
; AVX512-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
; AVX512-NEXT:    vinsertf64x4 $1, %ymm3, %zmm2, %zmm2
; AVX512-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512-NEXT:    vbroadcastsd {{.*#+}} zmm1 = [-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0]
; AVX512-NEXT:    vfmadd213pd {{.*#+}} zmm2 = (zmm0 * zmm2) + zmm1
; AVX512-NEXT:    vfmsub213pd {{.*#+}} zmm2 = (zmm0 * zmm2) - zmm1
; AVX512-NEXT:    vmovapd %zmm2, %zmm0
; AVX512-NEXT:    retq
  %l0 = call <4 x double> @llvm.fma.v4f32(<4 x double> %a2, <4 x double> %a0, <4 x double> splat (double -2.000000e+00))
  %h0 = call <4 x double> @llvm.fma.v4f32(<4 x double> %a3, <4 x double> %a1, <4 x double> splat (double -2.000000e+00))
  %l1 = call <4 x double> @llvm.fma.v4f32(<4 x double> %a0, <4 x double> %l0, <4 x double> splat (double +2.000000e+00))
  %h1 = call <4 x double> @llvm.fma.v4f32(<4 x double> %a1, <4 x double> %h0, <4 x double> splat (double +2.000000e+00))
  %r = shufflevector <4 x double> %l1, <4 x double> %h1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  ret <8 x double> %r
}

; FMA can't be concatenated until after max intrinsics have lowered, but then the v4f32 broadcasted constant is hidden behind an EXTRACT_SUBVECTOR
define <8 x float> @concat_fma_v8f32_v4f32_late_concat(<4 x float> %x, <4 x float> %y, <8 x float> %z) {
; FMA4-LABEL: concat_fma_v8f32_v4f32_late_concat:
; FMA4:       # %bb.0:
; FMA4-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
; FMA4-NEXT:    # ymm3 = mem[0,1,0,1]
; FMA4-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; FMA4-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; FMA4-NEXT:    vmaxps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
; FMA4-NEXT:    vfmaddps {{.*#+}} ymm0 = (ymm0 * ymm0) + ymm3
; FMA4-NEXT:    vmaxps %ymm3, %ymm0, %ymm0
; FMA4-NEXT:    vaddps %ymm2, %ymm0, %ymm0
; FMA4-NEXT:    retq
;
; AVX2-LABEL: concat_fma_v8f32_v4f32_late_concat:
; AVX2:       # %bb.0:
; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
; AVX2-NEXT:    vmaxps %ymm3, %ymm2, %ymm2
; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX2-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm0 * ymm0) + ymm3
; AVX2-NEXT:    vmaxps %ymm3, %ymm0, %ymm0
; AVX2-NEXT:    vaddps %ymm2, %ymm0, %ymm0
; AVX2-NEXT:    retq
;
; AVX512-LABEL: concat_fma_v8f32_v4f32_late_concat:
; AVX512:       # %bb.0:
; AVX512-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512-NEXT:    vmaxps %ymm3, %ymm2, %ymm2
; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX512-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm0 * ymm0) + ymm3
; AVX512-NEXT:    vmaxps %ymm3, %ymm0, %ymm0
; AVX512-NEXT:    vaddps %ymm2, %ymm0, %ymm0
; AVX512-NEXT:    retq
  %xx = call <4 x float> @llvm.fma.v4f32(<4 x float> %x, <4 x float> %x, <4 x float> splat (float 2.000000e+00))
  %yy = call <4 x float> @llvm.fma.v4f32(<4 x float> %y, <4 x float> %y, <4 x float> splat (float 2.000000e+00))
  %lo = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %xx, <4 x float> splat (float 2.000000e+00))
  %hi = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %yy, <4 x float> splat (float 2.000000e+00))
  %rhs = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> %z, <8 x float> splat (float 2.000000e+00))
  %lhs = shufflevector <4 x float> %lo, <4 x float> %hi, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  %add = fadd <8 x float> %lhs, %rhs
  ret <8 x float> %add
}

; negative - too many operands to concat
define <8 x float> @concat_fmadd_v8f32_v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %b0, <4 x float> %b1, <4 x float> %c0, <4 x float> %c1) {
; FMA4-LABEL: concat_fmadd_v8f32_v4f32:
; FMA4:       # %bb.0:
; FMA4-NEXT:    vfmaddps {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm4
; FMA4-NEXT:    vfmaddps {{.*#+}} xmm1 = (xmm1 * xmm3) + xmm5
; FMA4-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; FMA4-NEXT:    retq
;
; FMA3-LABEL: concat_fmadd_v8f32_v4f32:
; FMA3:       # %bb.0:
; FMA3-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; FMA3-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm2 * xmm0) + xmm4
; FMA3-NEXT:    vfmadd213ps {{.*#+}} xmm1 = (xmm3 * xmm1) + xmm5
; FMA3-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; FMA3-NEXT:    retq
  %v0 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a0, <4 x float> %b0, <4 x float> %c0)
  %v1 = call <4 x float> @llvm.fma.v4f32(<4 x float> %a1, <4 x float> %b1, <4 x float> %c1)
  %res = shufflevector <4 x float> %v0, <4 x float> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  ret <8 x float> %res
}