llvm/test/Transforms/LoopVectorize/early-exit-reductions-ordered.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -p loop-vectorize -force-vector-width=4 -hints-allow-reordering=false -force-ordered-reductions -S %s | FileCheck %s

; Strict (in-order) floating-point reductions in loops with uncountable early
; exits.

define float @fadd_ordered_reduction(ptr align 4 dereferenceable(4096) %src) {
; CHECK-LABEL: define float @fadd_ordered_reduction(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[SRC:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[RED:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[RED_NEXT:%.*]], %[[LOOP_LATCH]] ]
; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT:    [[CMP:%.*]] = fcmp uno float [[L]], 0.000000e+00
; CHECK-NEXT:    br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[RED_NEXT]] = fadd float [[RED]], [[L]]
; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    ret float 0.000000e+00
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[RED_NEXT_LCSSA:%.*]] = phi float [ [[RED_NEXT]], %[[LOOP_LATCH]] ]
; CHECK-NEXT:    ret float [[RED_NEXT_LCSSA]]
;
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %red = phi float [ 0.0, %entry ], [ %red.next, %loop.latch ]
  %gep = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv
  %l = load float, ptr %gep, align 4
  %cmp = fcmp uno float %l, 0.0
  br i1 %cmp, label %early.exit, label %loop.latch

loop.latch:
  %red.next = fadd float %red, %l
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop.header

early.exit:
  ret float 0.0

exit:
  ret float %red.next
}

; The reduction is used on the early exit. The latch exit does not use the
; reduction, as ordered reductions only allow a single use outside the loop.
define float @fadd_ordered_reduction_used_on_early_exit(ptr align 4 dereferenceable(4096) %src) {
; CHECK-LABEL: define float @fadd_ordered_reduction_used_on_early_exit(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[SRC:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[RED:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[RED_NEXT:%.*]], %[[LOOP_LATCH]] ]
; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT:    [[RED_NEXT]] = fadd float [[RED]], [[L]]
; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt float [[L]], 0.000000e+00
; CHECK-NEXT:    br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    [[RED_EARLY:%.*]] = phi float [ [[RED_NEXT]], %[[LOOP_HEADER]] ]
; CHECK-NEXT:    ret float [[RED_EARLY]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret float 0.000000e+00
;
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %red = phi float [ 0.0, %entry ], [ %red.next, %loop.latch ]
  %gep = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv
  %l = load float, ptr %gep, align 4
  %red.next = fadd float %red, %l
  %cmp = fcmp olt float %l, 0.0
  br i1 %cmp, label %early.exit, label %loop.latch

loop.latch:
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop.header

early.exit:
  %red.early = phi float [ %red.next, %loop.header ]
  ret float %red.early

exit:
  ret float 0.0
}

; The reduction phi is used on the early exit, so it cannot be computed as
; ordered reduction.
define float @fadd_ordered_reduction_phi_used_on_early_exit(ptr align 4 dereferenceable(4096) %src) {
; CHECK-LABEL: define float @fadd_ordered_reduction_phi_used_on_early_exit(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[SRC:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[RED:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[RED_NEXT:%.*]], %[[LOOP_LATCH]] ]
; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt float [[L]], 0.000000e+00
; CHECK-NEXT:    br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[RED_NEXT]] = fadd float [[RED]], [[L]]
; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    [[RED_EARLY:%.*]] = phi float [ [[RED]], %[[LOOP_HEADER]] ]
; CHECK-NEXT:    ret float [[RED_EARLY]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[RED_NEXT_LCSSA:%.*]] = phi float [ [[RED_NEXT]], %[[LOOP_LATCH]] ]
; CHECK-NEXT:    ret float [[RED_NEXT_LCSSA]]
;
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %red = phi float [ 0.0, %entry ], [ %red.next, %loop.latch ]
  %gep = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv
  %l = load float, ptr %gep, align 4
  %cmp = fcmp olt float %l, 0.0
  br i1 %cmp, label %early.exit, label %loop.latch

loop.latch:
  %red.next = fadd float %red, %l
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop.header

early.exit:
  %red.early = phi float [ %red, %loop.header ]
  ret float %red.early

exit:
  ret float %red.next
}

define float @fmaxnum_reduction(ptr align 4 dereferenceable(4096) %src) {
; CHECK-LABEL: define float @fmaxnum_reduction(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[SRC:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[RED:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[RED_NEXT:%.*]], %[[LOOP_LATCH]] ]
; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT:    [[CMP:%.*]] = fcmp uno float [[L]], 0.000000e+00
; CHECK-NEXT:    br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[RED_NEXT]] = call float @llvm.maxnum.f32(float [[RED]], float [[L]])
; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    ret float 0.000000e+00
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[RED_NEXT_LCSSA:%.*]] = phi float [ [[RED_NEXT]], %[[LOOP_LATCH]] ]
; CHECK-NEXT:    ret float [[RED_NEXT_LCSSA]]
;
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %red = phi float [ 0.0, %entry ], [ %red.next, %loop.latch ]
  %gep = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv
  %l = load float, ptr %gep, align 4
  %cmp = fcmp uno float %l, 0.0
  br i1 %cmp, label %early.exit, label %loop.latch

loop.latch:
  %red.next = call float @llvm.maxnum.f32(float %red, float %l)
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop.header

early.exit:
  ret float 0.0

exit:
  ret float %red.next
}