test/Transforms/SLPVectorizer/AMDGPU/elementwise-fma-operand1.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -slp-threshold=14 < %s | FileCheck %s
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -slp-threshold=14 < %s | FileCheck %s
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -slp-threshold=14 < %s | FileCheck %s
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -slp-threshold=12 < %s | FileCheck %s --check-prefix=THR12
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -slp-threshold=12 < %s | FileCheck %s --check-prefix=THR12
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -slp-threshold=12 < %s | FileCheck %s --check-prefix=THR12

; Elementwise d = c + a * b, where the fmul is operand 1 of the fadd. These
; targets halve the cost of a packed fmul, so SLP is tempted to vectorize and
; break the scalar fma chain. The 14 runs sit at the cost boundary. The 12 runs
; vectorize either way and guard against the fmuladd marking landing on the load
; at operand 0 after the fma detection picked the fmul at operand 1, which
; asserts. axpy4_mixed_reassoc carries reassoc on one lane only, so the whole
; bundle has to be reassociative before the search gives up on operand 1.

define void @axpy4_contract(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; CHECK-LABEL: define void @axpy4_contract(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
; CHECK-NEXT:    [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
; CHECK-NEXT:    [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
; CHECK-NEXT:    store <4 x float> [[TMP4]], ptr [[D]], align 4
; CHECK-NEXT:    ret void
;
; THR12-LABEL: define void @axpy4_contract(
; THR12-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
; THR12-NEXT:  [[ENTRY:.*:]]
; THR12-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
; THR12-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
; THR12-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
; THR12-NEXT:    [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
; THR12-NEXT:    [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
; THR12-NEXT:    store <4 x float> [[TMP4]], ptr [[D]], align 4
; THR12-NEXT:    ret void
;
entry:
  %c0 = load float, ptr %c, align 4
  %a0 = load float, ptr %a, align 4
  %b0 = load float, ptr %b, align 4
  %m0 = fmul contract float %a0, %b0
  %r0 = fadd contract float %c0, %m0
  store float %r0, ptr %d, align 4
  %cp1 = getelementptr inbounds float, ptr %c, i64 1
  %c1 = load float, ptr %cp1, align 4
  %ap1 = getelementptr inbounds float, ptr %a, i64 1
  %a1 = load float, ptr %ap1, align 4
  %bp1 = getelementptr inbounds float, ptr %b, i64 1
  %b1 = load float, ptr %bp1, align 4
  %m1 = fmul contract float %a1, %b1
  %r1 = fadd contract float %c1, %m1
  %dp1 = getelementptr inbounds float, ptr %d, i64 1
  store float %r1, ptr %dp1, align 4
  %cp2 = getelementptr inbounds float, ptr %c, i64 2
  %c2 = load float, ptr %cp2, align 4
  %ap2 = getelementptr inbounds float, ptr %a, i64 2
  %a2 = load float, ptr %ap2, align 4
  %bp2 = getelementptr inbounds float, ptr %b, i64 2
  %b2 = load float, ptr %bp2, align 4
  %m2 = fmul contract float %a2, %b2
  %r2 = fadd contract float %c2, %m2
  %dp2 = getelementptr inbounds float, ptr %d, i64 2
  store float %r2, ptr %dp2, align 4
  %cp3 = getelementptr inbounds float, ptr %c, i64 3
  %c3 = load float, ptr %cp3, align 4
  %ap3 = getelementptr inbounds float, ptr %a, i64 3
  %a3 = load float, ptr %ap3, align 4
  %bp3 = getelementptr inbounds float, ptr %b, i64 3
  %b3 = load float, ptr %bp3, align 4
  %m3 = fmul contract float %a3, %b3
  %r3 = fadd contract float %c3, %m3
  %dp3 = getelementptr inbounds float, ptr %d, i64 3
  store float %r3, ptr %dp3, align 4
  ret void
}

define void @axpy4_reassoc(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; CHECK-LABEL: define void @axpy4_reassoc(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
; CHECK-NEXT:    [[TMP3:%.*]] = fmul reassoc contract <4 x float> [[TMP1]], [[TMP2]]
; CHECK-NEXT:    [[TMP4:%.*]] = fadd reassoc contract <4 x float> [[TMP0]], [[TMP3]]
; CHECK-NEXT:    store <4 x float> [[TMP4]], ptr [[D]], align 4
; CHECK-NEXT:    ret void
;
; THR12-LABEL: define void @axpy4_reassoc(
; THR12-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; THR12-NEXT:  [[ENTRY:.*:]]
; THR12-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
; THR12-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
; THR12-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
; THR12-NEXT:    [[TMP3:%.*]] = fmul reassoc contract <4 x float> [[TMP1]], [[TMP2]]
; THR12-NEXT:    [[TMP4:%.*]] = fadd reassoc contract <4 x float> [[TMP0]], [[TMP3]]
; THR12-NEXT:    store <4 x float> [[TMP4]], ptr [[D]], align 4
; THR12-NEXT:    ret void
;
entry:
  %c0 = load float, ptr %c, align 4
  %a0 = load float, ptr %a, align 4
  %b0 = load float, ptr %b, align 4
  %m0 = fmul contract reassoc float %a0, %b0
  %r0 = fadd contract reassoc float %c0, %m0
  store float %r0, ptr %d, align 4
  %cp1 = getelementptr inbounds float, ptr %c, i64 1
  %c1 = load float, ptr %cp1, align 4
  %ap1 = getelementptr inbounds float, ptr %a, i64 1
  %a1 = load float, ptr %ap1, align 4
  %bp1 = getelementptr inbounds float, ptr %b, i64 1
  %b1 = load float, ptr %bp1, align 4
  %m1 = fmul contract reassoc float %a1, %b1
  %r1 = fadd contract reassoc float %c1, %m1
  %dp1 = getelementptr inbounds float, ptr %d, i64 1
  store float %r1, ptr %dp1, align 4
  %cp2 = getelementptr inbounds float, ptr %c, i64 2
  %c2 = load float, ptr %cp2, align 4
  %ap2 = getelementptr inbounds float, ptr %a, i64 2
  %a2 = load float, ptr %ap2, align 4
  %bp2 = getelementptr inbounds float, ptr %b, i64 2
  %b2 = load float, ptr %bp2, align 4
  %m2 = fmul contract reassoc float %a2, %b2
  %r2 = fadd contract reassoc float %c2, %m2
  %dp2 = getelementptr inbounds float, ptr %d, i64 2
  store float %r2, ptr %dp2, align 4
  %cp3 = getelementptr inbounds float, ptr %c, i64 3
  %c3 = load float, ptr %cp3, align 4
  %ap3 = getelementptr inbounds float, ptr %a, i64 3
  %a3 = load float, ptr %ap3, align 4
  %bp3 = getelementptr inbounds float, ptr %b, i64 3
  %b3 = load float, ptr %bp3, align 4
  %m3 = fmul contract reassoc float %a3, %b3
  %r3 = fadd contract reassoc float %c3, %m3
  %dp3 = getelementptr inbounds float, ptr %d, i64 3
  store float %r3, ptr %dp3, align 4
  ret void
}

define void @axpy4_mixed_reassoc(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; CHECK-LABEL: define void @axpy4_mixed_reassoc(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
; CHECK-NEXT:    [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
; CHECK-NEXT:    [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
; CHECK-NEXT:    store <4 x float> [[TMP4]], ptr [[D]], align 4
; CHECK-NEXT:    ret void
;
; THR12-LABEL: define void @axpy4_mixed_reassoc(
; THR12-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; THR12-NEXT:  [[ENTRY:.*:]]
; THR12-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[C]], align 4
; THR12-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[A]], align 4
; THR12-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[B]], align 4
; THR12-NEXT:    [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
; THR12-NEXT:    [[TMP4:%.*]] = fadd contract <4 x float> [[TMP0]], [[TMP3]]
; THR12-NEXT:    store <4 x float> [[TMP4]], ptr [[D]], align 4
; THR12-NEXT:    ret void
;
entry:
  %c0 = load float, ptr %c, align 4
  %a0 = load float, ptr %a, align 4
  %b0 = load float, ptr %b, align 4
  %m0 = fmul contract float %a0, %b0
  %r0 = fadd contract reassoc float %c0, %m0
  store float %r0, ptr %d, align 4
  %cp1 = getelementptr inbounds float, ptr %c, i64 1
  %c1 = load float, ptr %cp1, align 4
  %ap1 = getelementptr inbounds float, ptr %a, i64 1
  %a1 = load float, ptr %ap1, align 4
  %bp1 = getelementptr inbounds float, ptr %b, i64 1
  %b1 = load float, ptr %bp1, align 4
  %m1 = fmul contract float %a1, %b1
  %r1 = fadd contract float %c1, %m1
  %dp1 = getelementptr inbounds float, ptr %d, i64 1
  store float %r1, ptr %dp1, align 4
  %cp2 = getelementptr inbounds float, ptr %c, i64 2
  %c2 = load float, ptr %cp2, align 4
  %ap2 = getelementptr inbounds float, ptr %a, i64 2
  %a2 = load float, ptr %ap2, align 4
  %bp2 = getelementptr inbounds float, ptr %b, i64 2
  %b2 = load float, ptr %bp2, align 4
  %m2 = fmul contract float %a2, %b2
  %r2 = fadd contract float %c2, %m2
  %dp2 = getelementptr inbounds float, ptr %d, i64 2
  store float %r2, ptr %dp2, align 4
  %cp3 = getelementptr inbounds float, ptr %c, i64 3
  %c3 = load float, ptr %cp3, align 4
  %ap3 = getelementptr inbounds float, ptr %a, i64 3
  %a3 = load float, ptr %ap3, align 4
  %bp3 = getelementptr inbounds float, ptr %b, i64 3
  %b3 = load float, ptr %bp3, align 4
  %m3 = fmul contract float %a3, %b3
  %r3 = fadd contract float %c3, %m3
  %dp3 = getelementptr inbounds float, ptr %d, i64 3
  store float %r3, ptr %dp3, align 4
  ret void
}