llvm/test/CodeGen/NVPTX/nvptx-fold-fma.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt < %s -passes=nvptx-ir-peephole -S | FileCheck %s

target triple = "nvptx64-nvidia-cuda"

; fsub(fmul(a, b), c) => fma(a, b, fneg(c))
define float @test_fsub_fmul_c(float %a, float %b, float %c) {
; CHECK-LABEL: define float @test_fsub_fmul_c(
; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = fneg contract float [[C]]
; CHECK-NEXT:    [[TMP2:%.*]] = call contract float @llvm.fma.f32(float [[A]], float [[B]], float [[TMP1]])
; CHECK-NEXT:    ret float [[TMP2]]
;
  %mul = fmul contract float %a, %b
  %sub = fsub contract float %mul, %c
  ret float %sub
}


; fsub(c, fmul(a, b)) => fma(-a, b, c)
define float @test_fsub_c_fmul(float %a, float %b, float %c) {
; CHECK-LABEL: define float @test_fsub_c_fmul(
; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = fneg contract float [[A]]
; CHECK-NEXT:    [[TMP2:%.*]] = call contract float @llvm.fma.f32(float [[TMP1]], float [[B]], float [[C]])
; CHECK-NEXT:    ret float [[TMP2]]
;
  %mul = fmul contract float %a, %b
  %sub = fsub contract float %c, %mul
  ret float %sub
}


; fsub(fmul(a, b), fmul(c, d)) => fma(a, b, fneg(fmul(c, d)))
define float @test_fsub_fmul_fmul(float %a, float %b, float %c, float %d) {
; CHECK-LABEL: define float @test_fsub_fmul_fmul(
; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], float [[D:%.*]]) {
; CHECK-NEXT:    [[MUL2:%.*]] = fmul contract float [[C]], [[D]]
; CHECK-NEXT:    [[TMP1:%.*]] = fneg contract float [[MUL2]]
; CHECK-NEXT:    [[TMP2:%.*]] = call contract float @llvm.fma.f32(float [[A]], float [[B]], float [[TMP1]])
; CHECK-NEXT:    ret float [[TMP2]]
;
  %mul1 = fmul contract float %a, %b
  %mul2 = fmul contract float %c, %d
  %sub = fsub contract float %mul1, %mul2
  ret float %sub
}


; fsub(fmul(a, b), fmul(c, d)) => fma(fneg(c), d, fmul(a, b)))
; fmul(a, b) has multiple uses.
define float @test_fsub_fmul_fmul_multiple_use(float %a, float %b, float %c, float %d) {
; CHECK-LABEL: define float @test_fsub_fmul_fmul_multiple_use(
; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], float [[D:%.*]]) {
; CHECK-NEXT:    [[MUL1:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT:    [[TMP1:%.*]] = fneg contract float [[C]]
; CHECK-NEXT:    [[TMP2:%.*]] = call contract float @llvm.fma.f32(float [[TMP1]], float [[D]], float [[MUL1]])
; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP2]], [[MUL1]]
; CHECK-NEXT:    ret float [[ADD]]
;
  %mul1 = fmul contract float %a, %b
  %mul2 = fmul contract float %c, %d
  %sub = fsub contract float %mul1, %mul2
  %add = fadd float %sub, %mul1
  ret float %add
}


; fsub(fmul(a, b), c) => fma(a, b, fneg(c)) where fsub and fmul are in different BBs
define float @test_fsub_fmul_different_BB(float %a, float %b, float %c, i32 %n) {
; CHECK-LABEL: define float @test_fsub_fmul_different_BB(
; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT:  [[INIT:.*]]:
; CHECK-NEXT:    [[CMP_ITER:%.*]] = icmp sgt i32 [[N]], 10
; CHECK-NEXT:    br i1 [[CMP_ITER]], label %[[ITERATION:.*]], label %[[EXIT:.*]]
; CHECK:       [[ITERATION]]:
; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[INIT]] ], [ [[I_NEXT:%.*]], %[[ITERATION]] ]
; CHECK-NEXT:    [[ACC:%.*]] = phi float [ [[C]], %[[INIT]] ], [ [[ACC_NEXT:%.*]], %[[ITERATION]] ]
; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
; CHECK-NEXT:    [[ACC_NEXT]] = fadd contract float [[ACC]], 1.000000e+00
; CHECK-NEXT:    [[CMP_LOOP:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[CMP_LOOP]], label %[[ITERATION]], label %[[EXIT]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[C_PHI:%.*]] = phi float [ [[C]], %[[INIT]] ], [ [[ACC_NEXT]], %[[ITERATION]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = fneg contract float [[C_PHI]]
; CHECK-NEXT:    [[TMP1:%.*]] = call contract float @llvm.fma.f32(float [[A]], float [[B]], float [[TMP0]])
; CHECK-NEXT:    ret float [[TMP1]]
;
init:
  %mul = fmul contract float %a, %b
  %cmp_iter = icmp sgt i32 %n, 10
  br i1 %cmp_iter, label %iteration, label %exit

iteration:
  %i = phi i32 [ 0, %init ], [ %i_next, %iteration ]
  %acc = phi float [ %c, %init ], [ %acc_next, %iteration ]
  %i_next = add i32 %i, 1
  %acc_next = fadd contract float %acc, 1.0
  %cmp_loop = icmp slt i32 %i_next, %n
  br i1 %cmp_loop, label %iteration, label %exit

exit:
  %c_phi = phi float [ %c, %init ], [ %acc_next, %iteration ]
  %sub = fsub contract float %mul, %c_phi
  ret float %sub
}


; fadd(fmul(a, b), c) => fma(a, b, c)
define float @test_fadd_fmul_c(float %a, float %b, float %c) {
; CHECK-LABEL: define float @test_fadd_fmul_c(
; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = call contract float @llvm.fma.f32(float [[A]], float [[B]], float [[C]])
; CHECK-NEXT:    ret float [[TMP1]]
;
  %mul = fmul contract float %a, %b
  %add = fadd contract float %mul, %c
  ret float %add
}


; fadd(c, fmul(a, b)) => fma(a, b, c)
define float @test_fadd_c_fmul(float %a, float %b, float %c) {
; CHECK-LABEL: define float @test_fadd_c_fmul(
; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], float [[C:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = call contract float @llvm.fma.f32(float [[A]], float [[B]], float [[C]])
; CHECK-NEXT:    ret float [[TMP1]]
;
  %mul = fmul contract float %a, %b
  %add = fadd contract float %c, %mul
  ret float %add
}


; fadd(fmul(a, b), fmul(c, d)) => fma(a, b, fmul(c, d))
define float @test_fadd_fmul_fmul(float %a, float %b, float %c, float %d) {
; CHECK-LABEL: define float @test_fadd_fmul_fmul(
; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], float [[D:%.*]]) {
; CHECK-NEXT:    [[MUL2:%.*]] = fmul contract float [[C]], [[D]]
; CHECK-NEXT:    [[TMP1:%.*]] = call contract float @llvm.fma.f32(float [[A]], float [[B]], float [[MUL2]])
; CHECK-NEXT:    ret float [[TMP1]]
;
  %mul1 = fmul contract float %a, %b
  %mul2 = fmul contract float %c, %d
  %add = fadd contract float %mul1, %mul2
  ret float %add
}


; fadd(fmul(a, b), c) => fma(a, b, c) where fadd and fmul are in different BBs
define float @test_fadd_fmul_different_BB(float %a, float %b, float %c, i32 %n) {
; CHECK-LABEL: define float @test_fadd_fmul_different_BB(
; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT:  [[INIT:.*]]:
; CHECK-NEXT:    [[CMP_ITER:%.*]] = icmp sgt i32 [[N]], 10
; CHECK-NEXT:    br i1 [[CMP_ITER]], label %[[ITERATION:.*]], label %[[EXIT:.*]]
; CHECK:       [[ITERATION]]:
; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[INIT]] ], [ [[I_NEXT:%.*]], %[[ITERATION]] ]
; CHECK-NEXT:    [[ACC:%.*]] = phi float [ [[C]], %[[INIT]] ], [ [[ACC_NEXT:%.*]], %[[ITERATION]] ]
; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
; CHECK-NEXT:    [[ACC_NEXT]] = fadd contract float [[ACC]], 1.000000e+00
; CHECK-NEXT:    [[CMP_LOOP:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[CMP_LOOP]], label %[[ITERATION]], label %[[EXIT]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[C_PHI:%.*]] = phi float [ [[C]], %[[INIT]] ], [ [[ACC_NEXT]], %[[ITERATION]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = call contract float @llvm.fma.f32(float [[A]], float [[B]], float [[C_PHI]])
; CHECK-NEXT:    ret float [[TMP0]]
;
init:
  %mul = fmul contract float %a, %b
  %cmp_iter = icmp sgt i32 %n, 10
  br i1 %cmp_iter, label %iteration, label %exit

iteration:
  %i = phi i32 [ 0, %init ], [ %i_next, %iteration ]
  %acc = phi float [ %c, %init ], [ %acc_next, %iteration ]
  %i_next = add i32 %i, 1
  %acc_next = fadd contract float %acc, 1.0
  %cmp_loop = icmp slt i32 %i_next, %n
  br i1 %cmp_loop, label %iteration, label %exit

exit:
  %c_phi = phi float [ %c, %init ], [ %acc_next, %iteration ]
  %add = fadd contract float %mul, %c_phi
  ret float %add
}


; These scenarios shouldn't work.
; fadd(fpext(fmul(a, b)), c) => fma(fpext(a), fpext(b), c)
define double @test_fadd_fpext_fmul_c(float %a, float %b, double %c) {
; CHECK-LABEL: define double @test_fadd_fpext_fmul_c(
; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], double [[C:%.*]]) {
; CHECK-NEXT:    [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT:    [[EXT:%.*]] = fpext float [[MUL]] to double
; CHECK-NEXT:    [[ADD:%.*]] = fadd contract double [[EXT]], [[C]]
; CHECK-NEXT:    ret double [[ADD]]
;
  %mul = fmul contract float %a, %b
  %ext = fpext float %mul to double
  %add = fadd contract double %ext, %c
  ret double %add
}


; fadd(c, fpext(fmul(a, b))) => fma(fpext(a), fpext(b), c)
define double @test_fadd_c_fpext_fmul(float %a, float %b, double %c) {
; CHECK-LABEL: define double @test_fadd_c_fpext_fmul(
; CHECK-SAME: float [[A:%.*]], float [[B:%.*]], double [[C:%.*]]) {
; CHECK-NEXT:    [[MUL:%.*]] = fmul contract float [[A]], [[B]]
; CHECK-NEXT:    [[EXT:%.*]] = fpext float [[MUL]] to double
; CHECK-NEXT:    [[ADD:%.*]] = fadd contract double [[C]], [[EXT]]
; CHECK-NEXT:    ret double [[ADD]]
;
  %mul = fmul contract float %a, %b
  %ext = fpext float %mul to double
  %add = fadd contract double %c, %ext
  ret double %add
}


; Double precision tests
; fsub(fmul(a, b), c) => fma(a, b, fneg(c))
define double @test_fsub_fmul_c_double(double %a, double %b, double %c) {
; CHECK-LABEL: define double @test_fsub_fmul_c_double(
; CHECK-SAME: double [[A:%.*]], double [[B:%.*]], double [[C:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = fneg contract double [[C]]
; CHECK-NEXT:    [[TMP2:%.*]] = call contract double @llvm.fma.f64(double [[A]], double [[B]], double [[TMP1]])
; CHECK-NEXT:    ret double [[TMP2]]
;
  %mul = fmul contract double %a, %b
  %sub = fsub contract double %mul, %c
  ret double %sub
}


; fadd(fmul(a, b), c) => fma(a, b, c)
define double @test_fadd_fmul_c_double(double %a, double %b, double %c) {
; CHECK-LABEL: define double @test_fadd_fmul_c_double(
; CHECK-SAME: double [[A:%.*]], double [[B:%.*]], double [[C:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = call contract double @llvm.fma.f64(double [[A]], double [[B]], double [[C]])
; CHECK-NEXT:    ret double [[TMP1]]
;
  %mul = fmul contract double %a, %b
  %add = fadd contract double %mul, %c
  ret double %add
}