llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck %s
; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -slp-threshold=15 < %s | FileCheck %s --check-prefix=THR15

; The fma check has to read the multiply's own fast math flags. Reading the
; add's instead makes every multiply look contractable, so a fusion that the
; backend will not perform gets paid for. mixed_contract_mul carries contract
; on one lane only and moves at the default threshold.


define void @nocontract_mul(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; CHECK-LABEL: define void @nocontract_mul(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0
; CHECK-NEXT:    [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
; CHECK-NEXT:    [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
; CHECK-NEXT:    [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[AP0]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[BP0]], align 4
; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[CP0]], align 4
; CHECK-NEXT:    [[TMP3:%.*]] = fmul <2 x float> [[TMP0]], [[TMP1]]
; CHECK-NEXT:    [[TMP4:%.*]] = fadd contract <2 x float> [[TMP3]], [[TMP2]]
; CHECK-NEXT:    store <2 x float> [[TMP4]], ptr [[DP0]], align 4
; CHECK-NEXT:    [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
; CHECK-NEXT:    [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
; CHECK-NEXT:    [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
; CHECK-NEXT:    [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x float>, ptr [[AP2]], align 4
; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x float>, ptr [[BP2]], align 4
; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x float>, ptr [[CP2]], align 4
; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], [[TMP6]]
; CHECK-NEXT:    [[TMP9:%.*]] = fadd contract <2 x float> [[TMP8]], [[TMP7]]
; CHECK-NEXT:    store <2 x float> [[TMP9]], ptr [[DP2]], align 4
; CHECK-NEXT:    ret void
;
; THR15-LABEL: define void @nocontract_mul(
; THR15-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
; THR15-NEXT:  [[ENTRY:.*:]]
; THR15-NEXT:    [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0
; THR15-NEXT:    [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
; THR15-NEXT:    [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
; THR15-NEXT:    [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
; THR15-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[AP0]], align 4
; THR15-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[BP0]], align 4
; THR15-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[CP0]], align 4
; THR15-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[TMP0]], [[TMP1]]
; THR15-NEXT:    [[TMP4:%.*]] = fadd contract <4 x float> [[TMP3]], [[TMP2]]
; THR15-NEXT:    store <4 x float> [[TMP4]], ptr [[DP0]], align 4
; THR15-NEXT:    ret void
;
entry:
  %ap0 = getelementptr inbounds float, ptr %a, i64 0
  %bp0 = getelementptr inbounds float, ptr %b, i64 0
  %cp0 = getelementptr inbounds float, ptr %c, i64 0
  %dp0 = getelementptr inbounds float, ptr %d, i64 0
  %a0 = load float, ptr %ap0
  %b0 = load float, ptr %bp0
  %c0 = load float, ptr %cp0
  %m0 = fmul float %a0, %b0
  %r0 = fadd contract float %m0, %c0
  store float %r0, ptr %dp0
  %ap1 = getelementptr inbounds float, ptr %a, i64 1
  %bp1 = getelementptr inbounds float, ptr %b, i64 1
  %cp1 = getelementptr inbounds float, ptr %c, i64 1
  %dp1 = getelementptr inbounds float, ptr %d, i64 1
  %a1 = load float, ptr %ap1
  %b1 = load float, ptr %bp1
  %c1 = load float, ptr %cp1
  %m1 = fmul float %a1, %b1
  %r1 = fadd contract float %m1, %c1
  store float %r1, ptr %dp1
  %ap2 = getelementptr inbounds float, ptr %a, i64 2
  %bp2 = getelementptr inbounds float, ptr %b, i64 2
  %cp2 = getelementptr inbounds float, ptr %c, i64 2
  %dp2 = getelementptr inbounds float, ptr %d, i64 2
  %a2 = load float, ptr %ap2
  %b2 = load float, ptr %bp2
  %c2 = load float, ptr %cp2
  %m2 = fmul float %a2, %b2
  %r2 = fadd contract float %m2, %c2
  store float %r2, ptr %dp2
  %ap3 = getelementptr inbounds float, ptr %a, i64 3
  %bp3 = getelementptr inbounds float, ptr %b, i64 3
  %cp3 = getelementptr inbounds float, ptr %c, i64 3
  %dp3 = getelementptr inbounds float, ptr %d, i64 3
  %a3 = load float, ptr %ap3
  %b3 = load float, ptr %bp3
  %c3 = load float, ptr %cp3
  %m3 = fmul float %a3, %b3
  %r3 = fadd contract float %m3, %c3
  store float %r3, ptr %dp3
  ret void
}

define void @contract_mul(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; CHECK-LABEL: define void @contract_mul(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0
; CHECK-NEXT:    [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
; CHECK-NEXT:    [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
; CHECK-NEXT:    [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[AP0]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[BP0]], align 4
; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[CP0]], align 4
; CHECK-NEXT:    [[TMP3:%.*]] = fmul contract <2 x float> [[TMP0]], [[TMP1]]
; CHECK-NEXT:    [[TMP4:%.*]] = fadd contract <2 x float> [[TMP3]], [[TMP2]]
; CHECK-NEXT:    store <2 x float> [[TMP4]], ptr [[DP0]], align 4
; CHECK-NEXT:    [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
; CHECK-NEXT:    [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
; CHECK-NEXT:    [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
; CHECK-NEXT:    [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x float>, ptr [[AP2]], align 4
; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x float>, ptr [[BP2]], align 4
; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x float>, ptr [[CP2]], align 4
; CHECK-NEXT:    [[TMP8:%.*]] = fmul contract <2 x float> [[TMP5]], [[TMP6]]
; CHECK-NEXT:    [[TMP9:%.*]] = fadd contract <2 x float> [[TMP8]], [[TMP7]]
; CHECK-NEXT:    store <2 x float> [[TMP9]], ptr [[DP2]], align 4
; CHECK-NEXT:    ret void
;
; THR15-LABEL: define void @contract_mul(
; THR15-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; THR15-NEXT:  [[ENTRY:.*:]]
; THR15-NEXT:    [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0
; THR15-NEXT:    [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
; THR15-NEXT:    [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
; THR15-NEXT:    [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
; THR15-NEXT:    [[A0:%.*]] = load float, ptr [[AP0]], align 4
; THR15-NEXT:    [[B0:%.*]] = load float, ptr [[BP0]], align 4
; THR15-NEXT:    [[C0:%.*]] = load float, ptr [[CP0]], align 4
; THR15-NEXT:    [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
; THR15-NEXT:    [[R0:%.*]] = fadd contract float [[M0]], [[C0]]
; THR15-NEXT:    store float [[R0]], ptr [[DP0]], align 4
; THR15-NEXT:    [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
; THR15-NEXT:    [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
; THR15-NEXT:    [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
; THR15-NEXT:    [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
; THR15-NEXT:    [[A1:%.*]] = load float, ptr [[AP1]], align 4
; THR15-NEXT:    [[B1:%.*]] = load float, ptr [[BP1]], align 4
; THR15-NEXT:    [[C1:%.*]] = load float, ptr [[CP1]], align 4
; THR15-NEXT:    [[M1:%.*]] = fmul contract float [[A1]], [[B1]]
; THR15-NEXT:    [[R1:%.*]] = fadd contract float [[M1]], [[C1]]
; THR15-NEXT:    store float [[R1]], ptr [[DP1]], align 4
; THR15-NEXT:    [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
; THR15-NEXT:    [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
; THR15-NEXT:    [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
; THR15-NEXT:    [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
; THR15-NEXT:    [[A2:%.*]] = load float, ptr [[AP2]], align 4
; THR15-NEXT:    [[B2:%.*]] = load float, ptr [[BP2]], align 4
; THR15-NEXT:    [[C2:%.*]] = load float, ptr [[CP2]], align 4
; THR15-NEXT:    [[M2:%.*]] = fmul contract float [[A2]], [[B2]]
; THR15-NEXT:    [[R2:%.*]] = fadd contract float [[M2]], [[C2]]
; THR15-NEXT:    store float [[R2]], ptr [[DP2]], align 4
; THR15-NEXT:    [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
; THR15-NEXT:    [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
; THR15-NEXT:    [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
; THR15-NEXT:    [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
; THR15-NEXT:    [[A3:%.*]] = load float, ptr [[AP3]], align 4
; THR15-NEXT:    [[B3:%.*]] = load float, ptr [[BP3]], align 4
; THR15-NEXT:    [[C3:%.*]] = load float, ptr [[CP3]], align 4
; THR15-NEXT:    [[M3:%.*]] = fmul contract float [[A3]], [[B3]]
; THR15-NEXT:    [[R3:%.*]] = fadd contract float [[M3]], [[C3]]
; THR15-NEXT:    store float [[R3]], ptr [[DP3]], align 4
; THR15-NEXT:    ret void
;
entry:
  %ap0 = getelementptr inbounds float, ptr %a, i64 0
  %bp0 = getelementptr inbounds float, ptr %b, i64 0
  %cp0 = getelementptr inbounds float, ptr %c, i64 0
  %dp0 = getelementptr inbounds float, ptr %d, i64 0
  %a0 = load float, ptr %ap0
  %b0 = load float, ptr %bp0
  %c0 = load float, ptr %cp0
  %m0 = fmul contract float %a0, %b0
  %r0 = fadd contract float %m0, %c0
  store float %r0, ptr %dp0
  %ap1 = getelementptr inbounds float, ptr %a, i64 1
  %bp1 = getelementptr inbounds float, ptr %b, i64 1
  %cp1 = getelementptr inbounds float, ptr %c, i64 1
  %dp1 = getelementptr inbounds float, ptr %d, i64 1
  %a1 = load float, ptr %ap1
  %b1 = load float, ptr %bp1
  %c1 = load float, ptr %cp1
  %m1 = fmul contract float %a1, %b1
  %r1 = fadd contract float %m1, %c1
  store float %r1, ptr %dp1
  %ap2 = getelementptr inbounds float, ptr %a, i64 2
  %bp2 = getelementptr inbounds float, ptr %b, i64 2
  %cp2 = getelementptr inbounds float, ptr %c, i64 2
  %dp2 = getelementptr inbounds float, ptr %d, i64 2
  %a2 = load float, ptr %ap2
  %b2 = load float, ptr %bp2
  %c2 = load float, ptr %cp2
  %m2 = fmul contract float %a2, %b2
  %r2 = fadd contract float %m2, %c2
  store float %r2, ptr %dp2
  %ap3 = getelementptr inbounds float, ptr %a, i64 3
  %bp3 = getelementptr inbounds float, ptr %b, i64 3
  %cp3 = getelementptr inbounds float, ptr %c, i64 3
  %dp3 = getelementptr inbounds float, ptr %d, i64 3
  %a3 = load float, ptr %ap3
  %b3 = load float, ptr %bp3
  %c3 = load float, ptr %cp3
  %m3 = fmul contract float %a3, %b3
  %r3 = fadd contract float %m3, %c3
  store float %r3, ptr %dp3
  ret void
}

define void @mixed_contract_mul(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; CHECK-LABEL: define void @mixed_contract_mul(
; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0
; CHECK-NEXT:    [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
; CHECK-NEXT:    [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
; CHECK-NEXT:    [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[AP0]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[BP0]], align 4
; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[CP0]], align 4
; CHECK-NEXT:    [[TMP3:%.*]] = fmul <2 x float> [[TMP0]], [[TMP1]]
; CHECK-NEXT:    [[TMP4:%.*]] = fadd contract <2 x float> [[TMP3]], [[TMP2]]
; CHECK-NEXT:    store <2 x float> [[TMP4]], ptr [[DP0]], align 4
; CHECK-NEXT:    [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
; CHECK-NEXT:    [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
; CHECK-NEXT:    [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
; CHECK-NEXT:    [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x float>, ptr [[AP2]], align 4
; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x float>, ptr [[BP2]], align 4
; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x float>, ptr [[CP2]], align 4
; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], [[TMP6]]
; CHECK-NEXT:    [[TMP9:%.*]] = fadd contract <2 x float> [[TMP8]], [[TMP7]]
; CHECK-NEXT:    store <2 x float> [[TMP9]], ptr [[DP2]], align 4
; CHECK-NEXT:    ret void
;
; THR15-LABEL: define void @mixed_contract_mul(
; THR15-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
; THR15-NEXT:  [[ENTRY:.*:]]
; THR15-NEXT:    [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0
; THR15-NEXT:    [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
; THR15-NEXT:    [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
; THR15-NEXT:    [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
; THR15-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[AP0]], align 4
; THR15-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[BP0]], align 4
; THR15-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[CP0]], align 4
; THR15-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[TMP0]], [[TMP1]]
; THR15-NEXT:    [[TMP4:%.*]] = fadd contract <4 x float> [[TMP3]], [[TMP2]]
; THR15-NEXT:    store <4 x float> [[TMP4]], ptr [[DP0]], align 4
; THR15-NEXT:    ret void
;
entry:
  %ap0 = getelementptr inbounds float, ptr %a, i64 0
  %bp0 = getelementptr inbounds float, ptr %b, i64 0
  %cp0 = getelementptr inbounds float, ptr %c, i64 0
  %dp0 = getelementptr inbounds float, ptr %d, i64 0
  %a0 = load float, ptr %ap0
  %b0 = load float, ptr %bp0
  %c0 = load float, ptr %cp0
  %m0 = fmul contract float %a0, %b0
  %r0 = fadd contract float %m0, %c0
  store float %r0, ptr %dp0
  %ap1 = getelementptr inbounds float, ptr %a, i64 1
  %bp1 = getelementptr inbounds float, ptr %b, i64 1
  %cp1 = getelementptr inbounds float, ptr %c, i64 1
  %dp1 = getelementptr inbounds float, ptr %d, i64 1
  %a1 = load float, ptr %ap1
  %b1 = load float, ptr %bp1
  %c1 = load float, ptr %cp1
  %m1 = fmul float %a1, %b1
  %r1 = fadd contract float %m1, %c1
  store float %r1, ptr %dp1
  %ap2 = getelementptr inbounds float, ptr %a, i64 2
  %bp2 = getelementptr inbounds float, ptr %b, i64 2
  %cp2 = getelementptr inbounds float, ptr %c, i64 2
  %dp2 = getelementptr inbounds float, ptr %d, i64 2
  %a2 = load float, ptr %ap2
  %b2 = load float, ptr %bp2
  %c2 = load float, ptr %cp2
  %m2 = fmul float %a2, %b2
  %r2 = fadd contract float %m2, %c2
  store float %r2, ptr %dp2
  %ap3 = getelementptr inbounds float, ptr %a, i64 3
  %bp3 = getelementptr inbounds float, ptr %b, i64 3
  %cp3 = getelementptr inbounds float, ptr %c, i64 3
  %dp3 = getelementptr inbounds float, ptr %d, i64 3
  %a3 = load float, ptr %ap3
  %b3 = load float, ptr %bp3
  %c3 = load float, ptr %cp3
  %m3 = fmul float %a3, %b3
  %r3 = fadd contract float %m3, %c3
  store float %r3, ptr %dp3
  ret void
}