; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -passes=slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -passes=slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -passes=slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256

target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"

@src64 = common global [8 x double] zeroinitializer, align 64
@src32 = common global [16 x float] zeroinitializer, align 64
@dst64 = common global [8 x double] zeroinitializer, align 64
@dst32 = common global [16 x float] zeroinitializer, align 64

declare float @llvm.fabs.f32(float)
declare double @llvm.fabs.f64(double)

;
; FABS
;

define void @fabs_2f64() #0 {
; CHECK-LABEL: @fabs_2f64(
; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr @src64, align 8
; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP1]])
; CHECK-NEXT:    store <2 x double> [[TMP2]], ptr @dst64, align 8
; CHECK-NEXT:    ret void
;
  %a0 = load double, ptr @src64, align 8
  %a1 = load double, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 1), align 8
  %fabs0 = call double @llvm.fabs.f64(double %a0)
  %fabs1 = call double @llvm.fabs.f64(double %a1)
  store double %fabs0, ptr @dst64, align 8
  store double %fabs1, ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 1), align 8
  ret void
}

define void @fabs_4f64() #0 {
; SSE-LABEL: @fabs_4f64(
; SSE-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr @src64, align 8
; SSE-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP1]])
; SSE-NEXT:    store <2 x double> [[TMP2]], ptr @dst64, align 8
; SSE-NEXT:    [[TMP3:%.*]] = load <2 x double>, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 2), align 8
; SSE-NEXT:    [[TMP4:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP3]])
; SSE-NEXT:    store <2 x double> [[TMP4]], ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 2), align 8
; SSE-NEXT:    ret void
;
; AVX-LABEL: @fabs_4f64(
; AVX-NEXT:    [[TMP1:%.*]] = load <4 x double>, ptr @src64, align 8
; AVX-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.fabs.v4f64(<4 x double> [[TMP1]])
; AVX-NEXT:    store <4 x double> [[TMP2]], ptr @dst64, align 8
; AVX-NEXT:    ret void
;
  %a0 = load double, ptr @src64, align 8
  %a1 = load double, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 1), align 8
  %a2 = load double, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 2), align 8
  %a3 = load double, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 3), align 8
  %fabs0 = call double @llvm.fabs.f64(double %a0)
  %fabs1 = call double @llvm.fabs.f64(double %a1)
  %fabs2 = call double @llvm.fabs.f64(double %a2)
  %fabs3 = call double @llvm.fabs.f64(double %a3)
  store double %fabs0, ptr @dst64, align 8
  store double %fabs1, ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 1), align 8
  store double %fabs2, ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 2), align 8
  store double %fabs3, ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 3), align 8
  ret void
}

define void @fabs_8f64() #0 {
; SSE-LABEL: @fabs_8f64(
; SSE-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr @src64, align 4
; SSE-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP1]])
; SSE-NEXT:    store <2 x double> [[TMP2]], ptr @dst64, align 4
; SSE-NEXT:    [[TMP3:%.*]] = load <2 x double>, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 2), align 4
; SSE-NEXT:    [[TMP4:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP3]])
; SSE-NEXT:    store <2 x double> [[TMP4]], ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 2), align 4
; SSE-NEXT:    [[TMP5:%.*]] = load <2 x double>, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 4), align 4
; SSE-NEXT:    [[TMP6:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP5]])
; SSE-NEXT:    store <2 x double> [[TMP6]], ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 4), align 4
; SSE-NEXT:    [[TMP7:%.*]] = load <2 x double>, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 6), align 4
; SSE-NEXT:    [[TMP8:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP7]])
; SSE-NEXT:    store <2 x double> [[TMP8]], ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 6), align 4
; SSE-NEXT:    ret void
;
; AVX256-LABEL: @fabs_8f64(
; AVX256-NEXT:    [[TMP1:%.*]] = load <4 x double>, ptr @src64, align 4
; AVX256-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.fabs.v4f64(<4 x double> [[TMP1]])
; AVX256-NEXT:    store <4 x double> [[TMP2]], ptr @dst64, align 4
; AVX256-NEXT:    [[TMP3:%.*]] = load <4 x double>, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 4), align 4
; AVX256-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.fabs.v4f64(<4 x double> [[TMP3]])
; AVX256-NEXT:    store <4 x double> [[TMP4]], ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 4), align 4
; AVX256-NEXT:    ret void
;
; AVX512-LABEL: @fabs_8f64(
; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, ptr @src64, align 4
; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.fabs.v8f64(<8 x double> [[TMP1]])
; AVX512-NEXT:    store <8 x double> [[TMP2]], ptr @dst64, align 4
; AVX512-NEXT:    ret void
;
  %a0 = load double, ptr @src64, align 4
  %a1 = load double, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 1), align 4
  %a2 = load double, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 2), align 4
  %a3 = load double, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 3), align 4
  %a4 = load double, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 4), align 4
  %a5 = load double, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 5), align 4
  %a6 = load double, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 6), align 4
  %a7 = load double, ptr getelementptr inbounds ([8 x double], ptr @src64, i32 0, i64 7), align 4
  %fabs0 = call double @llvm.fabs.f64(double %a0)
  %fabs1 = call double @llvm.fabs.f64(double %a1)
  %fabs2 = call double @llvm.fabs.f64(double %a2)
  %fabs3 = call double @llvm.fabs.f64(double %a3)
  %fabs4 = call double @llvm.fabs.f64(double %a4)
  %fabs5 = call double @llvm.fabs.f64(double %a5)
  %fabs6 = call double @llvm.fabs.f64(double %a6)
  %fabs7 = call double @llvm.fabs.f64(double %a7)
  store double %fabs0, ptr @dst64, align 4
  store double %fabs1, ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 1), align 4
  store double %fabs2, ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 2), align 4
  store double %fabs3, ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 3), align 4
  store double %fabs4, ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 4), align 4
  store double %fabs5, ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 5), align 4
  store double %fabs6, ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 6), align 4
  store double %fabs7, ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 7), align 4
  ret void
}

define void @fabs_4f32() #0 {
; CHECK-LABEL: @fabs_4f32(
; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr @src32, align 4
; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP1]])
; CHECK-NEXT:    store <4 x float> [[TMP2]], ptr @dst32, align 4
; CHECK-NEXT:    ret void
;
  %a0 = load float, ptr @src32, align 4
  %a1 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 1), align 4
  %a2 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 2), align 4
  %a3 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 3), align 4
  %fabs0 = call float @llvm.fabs.f32(float %a0)
  %fabs1 = call float @llvm.fabs.f32(float %a1)
  %fabs2 = call float @llvm.fabs.f32(float %a2)
  %fabs3 = call float @llvm.fabs.f32(float %a3)
  store float %fabs0, ptr @dst32, align 4
  store float %fabs1, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 1), align 4
  store float %fabs2, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 2), align 4
  store float %fabs3, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 3), align 4
  ret void
}

define void @fabs_8f32() #0 {
; SSE-LABEL: @fabs_8f32(
; SSE-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr @src32, align 4
; SSE-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP1]])
; SSE-NEXT:    store <4 x float> [[TMP2]], ptr @dst32, align 4
; SSE-NEXT:    [[TMP3:%.*]] = load <4 x float>, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 4), align 4
; SSE-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP3]])
; SSE-NEXT:    store <4 x float> [[TMP4]], ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 4), align 4
; SSE-NEXT:    ret void
;
; AVX-LABEL: @fabs_8f32(
; AVX-NEXT:    [[TMP1:%.*]] = load <8 x float>, ptr @src32, align 4
; AVX-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[TMP1]])
; AVX-NEXT:    store <8 x float> [[TMP2]], ptr @dst32, align 4
; AVX-NEXT:    ret void
;
  %a0 = load float, ptr @src32, align 4
  %a1 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 1), align 4
  %a2 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 2), align 4
  %a3 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 3), align 4
  %a4 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 4), align 4
  %a5 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 5), align 4
  %a6 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 6), align 4
  %a7 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 7), align 4
  %fabs0 = call float @llvm.fabs.f32(float %a0)
  %fabs1 = call float @llvm.fabs.f32(float %a1)
  %fabs2 = call float @llvm.fabs.f32(float %a2)
  %fabs3 = call float @llvm.fabs.f32(float %a3)
  %fabs4 = call float @llvm.fabs.f32(float %a4)
  %fabs5 = call float @llvm.fabs.f32(float %a5)
  %fabs6 = call float @llvm.fabs.f32(float %a6)
  %fabs7 = call float @llvm.fabs.f32(float %a7)
  store float %fabs0, ptr @dst32, align 4
  store float %fabs1, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 1), align 4
  store float %fabs2, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 2), align 4
  store float %fabs3, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 3), align 4
  store float %fabs4, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 4), align 4
  store float %fabs5, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 5), align 4
  store float %fabs6, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 6), align 4
  store float %fabs7, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 7), align 4
  ret void
}

define void @fabs_16f32() #0 {
; SSE-LABEL: @fabs_16f32(
; SSE-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr @src32, align 4
; SSE-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP1]])
; SSE-NEXT:    store <4 x float> [[TMP2]], ptr @dst32, align 4
; SSE-NEXT:    [[TMP3:%.*]] = load <4 x float>, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 4), align 4
; SSE-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP3]])
; SSE-NEXT:    store <4 x float> [[TMP4]], ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 4), align 4
; SSE-NEXT:    [[TMP5:%.*]] = load <4 x float>, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 8), align 4
; SSE-NEXT:    [[TMP6:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP5]])
; SSE-NEXT:    store <4 x float> [[TMP6]], ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 8), align 4
; SSE-NEXT:    [[TMP7:%.*]] = load <4 x float>, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 12), align 4
; SSE-NEXT:    [[TMP8:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP7]])
; SSE-NEXT:    store <4 x float> [[TMP8]], ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 12), align 4
; SSE-NEXT:    ret void
;
; AVX256-LABEL: @fabs_16f32(
; AVX256-NEXT:    [[TMP1:%.*]] = load <8 x float>, ptr @src32, align 4
; AVX256-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[TMP1]])
; AVX256-NEXT:    store <8 x float> [[TMP2]], ptr @dst32, align 4
; AVX256-NEXT:    [[TMP3:%.*]] = load <8 x float>, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 8), align 4
; AVX256-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[TMP3]])
; AVX256-NEXT:    store <8 x float> [[TMP4]], ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 8), align 4
; AVX256-NEXT:    ret void
;
; AVX512-LABEL: @fabs_16f32(
; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x float>, ptr @src32, align 4
; AVX512-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.fabs.v16f32(<16 x float> [[TMP1]])
; AVX512-NEXT:    store <16 x float> [[TMP2]], ptr @dst32, align 4
; AVX512-NEXT:    ret void
;
  %a0  = load float, ptr @src32, align 4
  %a1  = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64  1), align 4
  %a2  = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64  2), align 4
  %a3  = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64  3), align 4
  %a4  = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64  4), align 4
  %a5  = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64  5), align 4
  %a6  = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64  6), align 4
  %a7  = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64  7), align 4
  %a8  = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64  8), align 4
  %a9  = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64  9), align 4
  %a10 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 10), align 4
  %a11 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 11), align 4
  %a12 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 12), align 4
  %a13 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 13), align 4
  %a14 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 14), align 4
  %a15 = load float, ptr getelementptr inbounds ([16 x float], ptr @src32, i32 0, i64 15), align 4
  %fabs0  = call float @llvm.fabs.f32(float %a0 )
  %fabs1  = call float @llvm.fabs.f32(float %a1 )
  %fabs2  = call float @llvm.fabs.f32(float %a2 )
  %fabs3  = call float @llvm.fabs.f32(float %a3 )
  %fabs4  = call float @llvm.fabs.f32(float %a4 )
  %fabs5  = call float @llvm.fabs.f32(float %a5 )
  %fabs6  = call float @llvm.fabs.f32(float %a6 )
  %fabs7  = call float @llvm.fabs.f32(float %a7 )
  %fabs8  = call float @llvm.fabs.f32(float %a8 )
  %fabs9  = call float @llvm.fabs.f32(float %a9 )
  %fabs10 = call float @llvm.fabs.f32(float %a10)
  %fabs11 = call float @llvm.fabs.f32(float %a11)
  %fabs12 = call float @llvm.fabs.f32(float %a12)
  %fabs13 = call float @llvm.fabs.f32(float %a13)
  %fabs14 = call float @llvm.fabs.f32(float %a14)
  %fabs15 = call float @llvm.fabs.f32(float %a15)
  store float %fabs0 , ptr @dst32, align 4
  store float %fabs1 , ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64  1), align 4
  store float %fabs2 , ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64  2), align 4
  store float %fabs3 , ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64  3), align 4
  store float %fabs4 , ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64  4), align 4
  store float %fabs5 , ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64  5), align 4
  store float %fabs6 , ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64  6), align 4
  store float %fabs7 , ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64  7), align 4
  store float %fabs8 , ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64  8), align 4
  store float %fabs9 , ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64  9), align 4
  store float %fabs10, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 10), align 4
  store float %fabs11, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 11), align 4
  store float %fabs12, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 12), align 4
  store float %fabs13, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 13), align 4
  store float %fabs14, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 14), align 4
  store float %fabs15, ptr getelementptr inbounds ([16 x float], ptr @dst32, i32 0, i64 15), align 4
  ret void
}

attributes #0 = { nounwind }