llvm/test/Transforms/LoopVectorize/early-exit-reductions-ordered.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 --check-globals=none
; RUN: opt -p loop-vectorize -force-vector-width=4 -hints-allow-reordering=false -force-ordered-reductions -S %s | FileCheck %s

; Strict (in-order) floating-point reductions in loops with uncountable early
; exits.

define float @fadd_ordered_reduction(ptr align 4 dereferenceable(4096) %src) {
; CHECK-LABEL: define float @fadd_ordered_reduction(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[SRC:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY_INTERIM]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP4]] = call float @llvm.vector.reduce.fadd.v4f32(float [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-NEXT:    br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK:       [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[VECTOR_EARLY_EXIT]]:
; CHECK-NEXT:    br label %[[EARLY_EXIT:.*]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    ret float 0.000000e+00
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret float [[TMP4]]
;
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %red = phi float [ 0.0, %entry ], [ %red.next, %loop.latch ]
  %gep = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv
  %l = load float, ptr %gep, align 4
  %cmp = fcmp uno float %l, 0.0
  br i1 %cmp, label %early.exit, label %loop.latch

loop.latch:
  %red.next = fadd float %red, %l
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop.header

early.exit:
  ret float 0.0

exit:
  ret float %red.next
}

; The reduction is used on the early exit. The latch exit does not use the
; reduction, as ordered reductions only allow a single use outside the loop.
define float @fadd_ordered_reduction_used_on_early_exit(ptr align 4 dereferenceable(4096) %src) {
; CHECK-LABEL: define float @fadd_ordered_reduction_used_on_early_exit(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[SRC:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[RED:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[RED_NEXT:%.*]], %[[LOOP_LATCH]] ]
; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT:    [[RED_NEXT]] = fadd float [[RED]], [[L]]
; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt float [[L]], 0.000000e+00
; CHECK-NEXT:    br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    [[RED_EARLY:%.*]] = phi float [ [[RED_NEXT]], %[[LOOP_HEADER]] ]
; CHECK-NEXT:    ret float [[RED_EARLY]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret float 0.000000e+00
;
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %red = phi float [ 0.0, %entry ], [ %red.next, %loop.latch ]
  %gep = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv
  %l = load float, ptr %gep, align 4
  %red.next = fadd float %red, %l
  %cmp = fcmp olt float %l, 0.0
  br i1 %cmp, label %early.exit, label %loop.latch

loop.latch:
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop.header

early.exit:
  %red.early = phi float [ %red.next, %loop.header ]
  ret float %red.early

exit:
  ret float 0.0
}

; The reduction phi is used on the early exit, so it cannot be computed as
; ordered reduction.
define float @fadd_ordered_reduction_phi_used_on_early_exit(ptr align 4 dereferenceable(4096) %src) {
; CHECK-LABEL: define float @fadd_ordered_reduction_phi_used_on_early_exit(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[SRC:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[RED:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[RED_NEXT:%.*]], %[[LOOP_LATCH]] ]
; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt float [[L]], 0.000000e+00
; CHECK-NEXT:    br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[RED_NEXT]] = fadd float [[RED]], [[L]]
; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    [[RED_EARLY:%.*]] = phi float [ [[RED]], %[[LOOP_HEADER]] ]
; CHECK-NEXT:    ret float [[RED_EARLY]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[RED_NEXT_LCSSA:%.*]] = phi float [ [[RED_NEXT]], %[[LOOP_LATCH]] ]
; CHECK-NEXT:    ret float [[RED_NEXT_LCSSA]]
;
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %red = phi float [ 0.0, %entry ], [ %red.next, %loop.latch ]
  %gep = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv
  %l = load float, ptr %gep, align 4
  %cmp = fcmp olt float %l, 0.0
  br i1 %cmp, label %early.exit, label %loop.latch

loop.latch:
  %red.next = fadd float %red, %l
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop.header

early.exit:
  %red.early = phi float [ %red, %loop.header ]
  ret float %red.early

exit:
  ret float %red.next
}

define float @fmaxnum_reduction(ptr align 4 dereferenceable(4096) %src) {
; CHECK-LABEL: define float @fmaxnum_reduction(
; CHECK-SAME: ptr align 4 dereferenceable(4096) [[SRC:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[RED:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[RED_NEXT:%.*]], %[[LOOP_LATCH]] ]
; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP]], align 4
; CHECK-NEXT:    [[CMP:%.*]] = fcmp uno float [[L]], 0.000000e+00
; CHECK-NEXT:    br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[RED_NEXT]] = call float @llvm.maxnum.f32(float [[RED]], float [[L]])
; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    ret float 0.000000e+00
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[RED_NEXT_LCSSA:%.*]] = phi float [ [[RED_NEXT]], %[[LOOP_LATCH]] ]
; CHECK-NEXT:    ret float [[RED_NEXT_LCSSA]]
;
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %red = phi float [ 0.0, %entry ], [ %red.next, %loop.latch ]
  %gep = getelementptr inbounds nuw [4 x i8], ptr %src, i64 %iv
  %l = load float, ptr %gep, align 4
  %cmp = fcmp uno float %l, 0.0
  br i1 %cmp, label %early.exit, label %loop.latch

loop.latch:
  %red.next = call float @llvm.maxnum.f32(float %red, float %l)
  %iv.next = add nuw nsw i64 %iv, 1
  %ec = icmp eq i64 %iv.next, 1024
  br i1 %ec, label %exit, label %loop.header

early.exit:
  ret float 0.0

exit:
  ret float %red.next
}