Google GitGoogle Git
Sign in
llvm/llvm-project/e14b11b74dcafea1e2a71b9788b85b34ab4cf1e6/./llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions-min-max.ll
blob: d5a49323b191777af7a6c90fdc2b2aff560713c3 [file]
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -p loop-unroll -unroll-add-parallel-reductions -S %s | FileCheck %s
define i32 @test_smin_reduction(ptr %a, i64 %n) {
; CHECK-LABEL: define i32 @test_smin_reduction(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_1:%.*]] = phi i32 [ 2147483647, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN:%.*]] = phi i32 [ 2147483647, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[GEP_A]], align 4
; CHECK-NEXT: [[RDX_NEXT]] = call i32 @llvm.smin.i32(i32 [[MIN]], i32 [[TMP2]])
; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP_A_1]], align 4
; CHECK-NEXT: [[RDX_NEXT_1]] = call i32 @llvm.smin.i32(i32 [[MIN_1]], i32 [[TMP3]])
; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[EXIT_UNR_LCSSA]]:
; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_UNR:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call i32 @llvm.smin.i32(i32 [[RDX_NEXT]], i32 [[RDX_NEXT_1]])
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[MIN_EPIL_INIT:%.*]] = phi i32 [ 2147483647, %[[ENTRY]] ], [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
; CHECK: [[LOOP_EPIL]]:
; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[IV_EPIL_INIT]]
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 4
; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = call i32 @llvm.smin.i32(i32 [[MIN_EPIL_INIT]], i32 [[TMP4]])
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%min = phi i32 [ 2147483647, %entry ], [ %rdx.next, %loop ]
%gep.a = getelementptr inbounds nuw [4 x i8], ptr %a, i64 %iv
%0 = load i32, ptr %gep.a, align 4
%rdx.next = call i32 @llvm.smin.i32(i32 %min, i32 %0)
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop, !llvm.loop !0
exit:
%res = phi i32 [ %rdx.next, %loop ]
ret i32 %res
}
define i32 @test_smax_reduction(ptr %a, i64 %n) {
; CHECK-LABEL: define i32 @test_smax_reduction(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX_1:%.*]] = phi i32 [ -2147483648, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX:%.*]] = phi i32 [ -2147483648, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[GEP_A]], align 4
; CHECK-NEXT: [[RDX_NEXT]] = call i32 @llvm.smax.i32(i32 [[MAX]], i32 [[TMP2]])
; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP_A_1]], align 4
; CHECK-NEXT: [[RDX_NEXT_1]] = call i32 @llvm.smax.i32(i32 [[MAX_1]], i32 [[TMP3]])
; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP2:![0-9]+]]
; CHECK: [[EXIT_UNR_LCSSA]]:
; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX_UNR:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[RDX_NEXT]], i32 [[RDX_NEXT_1]])
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[MAX_EPIL_INIT:%.*]] = phi i32 [ -2147483648, %[[ENTRY]] ], [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
; CHECK: [[LOOP_EPIL]]:
; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[A]], i64 [[IV_EPIL_INIT]]
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 4
; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = call i32 @llvm.smax.i32(i32 [[MAX_EPIL_INIT]], i32 [[TMP4]])
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%max = phi i32 [ -2147483648, %entry ], [ %rdx.next, %loop ]
%gep.a = getelementptr inbounds nuw [4 x i8], ptr %a, i64 %iv
%0 = load i32, ptr %gep.a, align 4
%rdx.next = call i32 @llvm.smax.i32(i32 %max, i32 %0)
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop, !llvm.loop !0
exit:
%res = phi i32 [ %rdx.next, %loop ]
ret i32 %res
}
; -------------------------------------------------------------
; floating-point min and max reductions without fast-math flags.
; These are chained together sequentially.
; -------------------------------------------------------------
define float @test_fmin_reduction(ptr %a, i64 %n) {
; CHECK-LABEL: define float @test_fmin_reduction(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN:%.*]] = phi float [ +inf, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[GEP_A]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[MIN]], [[TMP2]]
; CHECK-NEXT: [[RDX_NEXT:%.*]] = select i1 [[CMP]], float [[MIN]], float [[TMP2]]
; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP_A_1]], align 4
; CHECK-NEXT: [[CMP_1:%.*]] = fcmp olt float [[RDX_NEXT]], [[TMP3]]
; CHECK-NEXT: [[RDX_NEXT_1]] = select i1 [[CMP_1]], float [[RDX_NEXT]], float [[TMP3]]
; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT_UNR_LCSSA]]:
; CHECK-NEXT: [[RES_PH:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_UNR:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[MIN_EPIL_INIT:%.*]] = phi float [ +inf, %[[ENTRY]] ], [ [[MIN_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
; CHECK: [[LOOP_EPIL]]:
; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV_EPIL_INIT]]
; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP_A_EPIL]], align 4
; CHECK-NEXT: [[CMP_EPIL:%.*]] = fcmp olt float [[MIN_EPIL_INIT]], [[TMP4]]
; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = select i1 [[CMP_EPIL]], float [[MIN_EPIL_INIT]], float [[TMP4]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi float [ [[RES_PH]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
; CHECK-NEXT: ret float [[RES]]
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%min = phi float [ 0x7FF0000000000000, %entry ], [ %rdx.next, %loop ]
%gep.a = getelementptr inbounds float, ptr %a, i64 %iv
%1 = load float, ptr %gep.a, align 4
%cmp = fcmp olt float %min, %1
%rdx.next = select i1 %cmp, float %min, float %1
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop, !llvm.loop !0
exit:
%res = phi float [ %rdx.next, %loop ]
ret float %res
}
define float @test_fmax_reduction(ptr %a, i64 %n) {
; CHECK-LABEL: define float @test_fmax_reduction(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX:%.*]] = phi float [ -inf, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[GEP_A]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt float [[MAX]], [[TMP2]]
; CHECK-NEXT: [[RDX_NEXT:%.*]] = select i1 [[CMP]], float [[MAX]], float [[TMP2]]
; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP_A_1]], align 4
; CHECK-NEXT: [[CMP_1:%.*]] = fcmp ogt float [[RDX_NEXT]], [[TMP3]]
; CHECK-NEXT: [[RDX_NEXT_1]] = select i1 [[CMP_1]], float [[RDX_NEXT]], float [[TMP3]]
; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[EXIT_UNR_LCSSA]]:
; CHECK-NEXT: [[RES_PH:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX_UNR:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[MAX_EPIL_INIT:%.*]] = phi float [ -inf, %[[ENTRY]] ], [ [[MAX_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
; CHECK: [[LOOP_EPIL]]:
; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV_EPIL_INIT]]
; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP_A_EPIL]], align 4
; CHECK-NEXT: [[CMP_EPIL:%.*]] = fcmp ogt float [[MAX_EPIL_INIT]], [[TMP4]]
; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = select i1 [[CMP_EPIL]], float [[MAX_EPIL_INIT]], float [[TMP4]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi float [ [[RES_PH]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
; CHECK-NEXT: ret float [[RES]]
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%max = phi float [ 0xFFF0000000000000, %entry ], [ %rdx.next, %loop ]
%gep.a = getelementptr inbounds float, ptr %a, i64 %iv
%1 = load float, ptr %gep.a, align 4
%cmp = fcmp ogt float %max, %1
%rdx.next = select i1 %cmp, float %max, float %1
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop, !llvm.loop !0
exit:
%res = phi float [ %rdx.next, %loop ]
ret float %res
}
; -------------------------------------------------------------
; floating-point min and max reductions with fast-math flags.
; -------------------------------------------------------------
define float @test_fast_math_fmin_reduction(ptr %a, i64 %n) {
; CHECK-LABEL: define float @test_fast_math_fmin_reduction(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_1:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN:%.*]] = phi float [ +inf, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[GEP_A]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ninf nsz olt float [[MIN]], [[TMP2]]
; CHECK-NEXT: [[RDX_NEXT]] = select nsz i1 [[CMP]], float [[MIN]], float [[TMP2]]
; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP_A_1]], align 4
; CHECK-NEXT: [[CMP_1:%.*]] = fcmp nnan ninf nsz olt float [[MIN_1]], [[TMP3]]
; CHECK-NEXT: [[RDX_NEXT_1]] = select nsz i1 [[CMP_1]], float [[MIN_1]], float [[TMP3]]
; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT_UNR_LCSSA]]:
; CHECK-NEXT: [[RES_PH:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_UNR:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp nnan ninf nsz olt float [[RDX_NEXT]], [[RDX_NEXT_1]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select nnan ninf nsz i1 [[RDX_MINMAX_CMP]], float [[RDX_NEXT]], float [[RDX_NEXT_1]]
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[MIN_EPIL_INIT:%.*]] = phi float [ +inf, %[[ENTRY]] ], [ [[RDX_MINMAX_SELECT]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
; CHECK: [[LOOP_EPIL]]:
; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV_EPIL_INIT]]
; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP_A_EPIL]], align 4
; CHECK-NEXT: [[CMP_EPIL:%.*]] = fcmp nnan ninf nsz olt float [[MIN_EPIL_INIT]], [[TMP4]]
; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = select nnan ninf nsz i1 [[CMP_EPIL]], float [[MIN_EPIL_INIT]], float [[TMP4]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi float [ [[RDX_MINMAX_SELECT]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
; CHECK-NEXT: ret float [[RES]]
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%min = phi float [ 0x7FF0000000000000, %entry ], [ %rdx.next, %loop ]
%gep.a = getelementptr inbounds float, ptr %a, i64 %iv
%1 = load float, ptr %gep.a, align 4
%cmp = fcmp nnan ninf nsz olt float %min, %1
%rdx.next = select nnan ninf nsz i1 %cmp, float %min, float %1
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop, !llvm.loop !0
exit:
%res = phi float [ %rdx.next, %loop ]
ret float %res
}
define float @test_fast_math_fmax_reduction(ptr %a, i64 %n) {
; CHECK-LABEL: define float @test_fast_math_fmax_reduction(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX_1:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX:%.*]] = phi float [ -inf, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[GEP_A]], align 4
; CHECK-NEXT: [[CMP:%.*]] = fcmp nnan ninf nsz ogt float [[MAX]], [[TMP2]]
; CHECK-NEXT: [[RDX_NEXT]] = select nsz i1 [[CMP]], float [[MAX]], float [[TMP2]]
; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP_A_1]], align 4
; CHECK-NEXT: [[CMP_1:%.*]] = fcmp nnan ninf nsz ogt float [[MAX_1]], [[TMP3]]
; CHECK-NEXT: [[RDX_NEXT_1]] = select nsz i1 [[CMP_1]], float [[MAX_1]], float [[TMP3]]
; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[EXIT_UNR_LCSSA]]:
; CHECK-NEXT: [[RES_PH:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX_UNR:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp nnan ninf nsz ogt float [[RDX_NEXT]], [[RDX_NEXT_1]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select nnan ninf nsz i1 [[RDX_MINMAX_CMP]], float [[RDX_NEXT]], float [[RDX_NEXT_1]]
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[MAX_EPIL_INIT:%.*]] = phi float [ -inf, %[[ENTRY]] ], [ [[RDX_MINMAX_SELECT]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
; CHECK: [[LOOP_EPIL]]:
; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV_EPIL_INIT]]
; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP_A_EPIL]], align 4
; CHECK-NEXT: [[CMP_EPIL:%.*]] = fcmp nnan ninf nsz ogt float [[MAX_EPIL_INIT]], [[TMP4]]
; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = select nnan ninf nsz i1 [[CMP_EPIL]], float [[MAX_EPIL_INIT]], float [[TMP4]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi float [ [[RDX_MINMAX_SELECT]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
; CHECK-NEXT: ret float [[RES]]
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%max = phi float [ 0xFFF0000000000000, %entry ], [ %rdx.next, %loop ]
%gep.a = getelementptr inbounds float, ptr %a, i64 %iv
%1 = load float, ptr %gep.a, align 4
%cmp = fcmp nnan ninf nsz ogt float %max, %1
%rdx.next = select nnan ninf nsz i1 %cmp, float %max, float %1
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop, !llvm.loop !0
exit:
%res = phi float [ %rdx.next, %loop ]
ret float %res
}
; -------------------------------------------------------------
; floating-point min and max reductions using intrinsics.
; TODO: Support parallel reductions for these as they are chained together.
; -------------------------------------------------------------
define float @test_fast_math_fmin_reduction_intrinsic(ptr %a, i64 %n) {
; CHECK-LABEL: define float @test_fast_math_fmin_reduction_intrinsic(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_1:%.*]] = phi float [ f0x7F7FFFFF, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN:%.*]] = phi float [ +inf, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[GEP_A]], align 4
; CHECK-NEXT: [[RDX_NEXT]] = call nsz float @llvm.minnum.f32(float [[MIN]], float [[TMP2]])
; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP_A_1]], align 4
; CHECK-NEXT: [[RDX_NEXT_1]] = call nsz float @llvm.minnum.f32(float [[MIN_1]], float [[TMP3]])
; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT_UNR_LCSSA]]:
; CHECK-NEXT: [[RES_PH:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_UNR:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp nnan ninf nsz olt float [[RDX_NEXT]], [[RDX_NEXT_1]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select nnan ninf nsz i1 [[RDX_MINMAX_CMP]], float [[RDX_NEXT]], float [[RDX_NEXT_1]]
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[MIN_EPIL_INIT:%.*]] = phi float [ +inf, %[[ENTRY]] ], [ [[RDX_MINMAX_SELECT]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
; CHECK: [[LOOP_EPIL]]:
; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV_EPIL_INIT]]
; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP_A_EPIL]], align 4
; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = call nnan ninf nsz float @llvm.minnum.f32(float [[MIN_EPIL_INIT]], float [[TMP4]])
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi float [ [[RDX_MINMAX_SELECT]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
; CHECK-NEXT: ret float [[RES]]
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%min = phi float [ 0x7FF0000000000000, %entry ], [ %rdx.next, %loop ]
%gep.a = getelementptr inbounds float, ptr %a, i64 %iv
%1 = load float, ptr %gep.a, align 4
%rdx.next = call nnan ninf nsz float @llvm.minnum.f32(float %min, float %1)
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop, !llvm.loop !0
exit:
%res = phi float [ %rdx.next, %loop ]
ret float %res
}
define float @test_fast_math_fmax_reduction_intrinsic(ptr %a, i64 %n) {
; CHECK-LABEL: define float @test_fast_math_fmax_reduction_intrinsic(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX_1:%.*]] = phi float [ f0xFF7FFFFF, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX:%.*]] = phi float [ -inf, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[GEP_A]], align 4
; CHECK-NEXT: [[RDX_NEXT]] = call nsz float @llvm.maxnum.f32(float [[MAX]], float [[TMP2]])
; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP_A_1]], align 4
; CHECK-NEXT: [[RDX_NEXT_1]] = call nsz float @llvm.maxnum.f32(float [[MAX_1]], float [[TMP3]])
; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT_UNR_LCSSA]]:
; CHECK-NEXT: [[RES_PH:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX_UNR:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp nnan ninf nsz ogt float [[RDX_NEXT]], [[RDX_NEXT_1]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select nnan ninf nsz i1 [[RDX_MINMAX_CMP]], float [[RDX_NEXT]], float [[RDX_NEXT_1]]
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[MAX_EPIL_INIT:%.*]] = phi float [ -inf, %[[ENTRY]] ], [ [[RDX_MINMAX_SELECT]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
; CHECK: [[LOOP_EPIL]]:
; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV_EPIL_INIT]]
; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP_A_EPIL]], align 4
; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = call nnan ninf nsz float @llvm.maxnum.f32(float [[MAX_EPIL_INIT]], float [[TMP4]])
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi float [ [[RDX_MINMAX_SELECT]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
; CHECK-NEXT: ret float [[RES]]
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%max = phi float [ 0xFFF0000000000000, %entry ], [ %rdx.next, %loop ]
%gep.a = getelementptr inbounds float, ptr %a, i64 %iv
%1 = load float, ptr %gep.a, align 4
%rdx.next = call nnan ninf nsz float @llvm.maxnum.f32(float %max, float %1)
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop, !llvm.loop !0
exit:
%res = phi float [ %rdx.next, %loop ]
ret float %res
}
define <4 x float> @test_vector_fmin_reduction_intrinsic(ptr %a, i64 %n) {
; CHECK-LABEL: define <4 x float> @test_vector_fmin_reduction_intrinsic(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX:%.*]] = phi <4 x float> [ splat (float +inf), %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds nuw <4 x float>, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[GEP_A]], align 16
; CHECK-NEXT: [[RDX_NEXT:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX]], <4 x float> [[TMP2]])
; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds nuw <4 x float>, ptr [[A]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[TMP3:%.*]] = load <4 x float>, ptr [[GEP_A_1]], align 16
; CHECK-NEXT: [[RDX_NEXT_1]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_NEXT]], <4 x float> [[TMP3]])
; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT_UNR_LCSSA]]:
; CHECK-NEXT: [[RES_PH:%.*]] = phi <4 x float> [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX_UNR:%.*]] = phi <4 x float> [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[RDX_EPIL_INIT:%.*]] = phi <4 x float> [ splat (float +inf), %[[ENTRY]] ], [ [[RDX_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
; CHECK: [[LOOP_EPIL]]:
; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw <4 x float>, ptr [[A]], i64 [[IV_EPIL_INIT]]
; CHECK-NEXT: [[TMP4:%.*]] = load <4 x float>, ptr [[GEP_A_EPIL]], align 16
; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[RDX_EPIL_INIT]], <4 x float> [[TMP4]])
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi <4 x float> [ [[RES_PH]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
; CHECK-NEXT: ret <4 x float> [[RES]]
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%rdx = phi <4 x float> [ <float 0x7FF0000000000000, float 0x7FF0000000000000, float 0x7FF0000000000000, float 0x7FF0000000000000>, %entry ], [ %rdx.next, %loop ]
%gep.a = getelementptr inbounds nuw <4 x float>, ptr %a, i64 %iv
%1 = load <4 x float>, ptr %gep.a, align 16
%rdx.next = call <4 x float> @llvm.minnum.v4f32(<4 x float> %rdx, <4 x float> %1)
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop, !llvm.loop !0
exit:
%res = phi <4 x float> [ %rdx.next, %loop ]
ret <4 x float> %res
}
; -------------------------------------------------------------
; integer min and max reductions using intrinsics.
; -------------------------------------------------------------
; TODO: chained reduction; parallelize this case.
define i32 @test_smin_reduction_intrinsic(ptr %a, i64 %n) {
; CHECK-LABEL: define i32 @test_smin_reduction_intrinsic(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_1:%.*]] = phi i32 [ 2147483647, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN:%.*]] = phi i32 [ 2147483647, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[GEP_A]], align 4
; CHECK-NEXT: [[RDX_NEXT]] = call i32 @llvm.smin.i32(i32 [[MIN]], i32 [[TMP2]])
; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP_A_1]], align 4
; CHECK-NEXT: [[RDX_NEXT_1]] = call i32 @llvm.smin.i32(i32 [[MIN_1]], i32 [[TMP3]])
; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[EXIT_UNR_LCSSA]]:
; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_UNR:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call i32 @llvm.smin.i32(i32 [[RDX_NEXT]], i32 [[RDX_NEXT_1]])
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[MIN_EPIL_INIT:%.*]] = phi i32 [ 2147483647, %[[ENTRY]] ], [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
; CHECK: [[LOOP_EPIL]]:
; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_EPIL_INIT]]
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 4
; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = call i32 @llvm.smin.i32(i32 [[MIN_EPIL_INIT]], i32 [[TMP4]])
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%min = phi i32 [ 2147483647, %entry ], [ %rdx.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
%1 = load i32, ptr %gep.a, align 4
%rdx.next = call i32 @llvm.smin.i32(i32 %min, i32 %1)
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop, !llvm.loop !0
exit:
%res = phi i32 [ %rdx.next, %loop ]
ret i32 %res
}
; TODO: chained reduction; parallelize this case.
define i32 @test_smax_reduction_intrinsic(ptr %a, i64 %n) {
; CHECK-LABEL: define i32 @test_smax_reduction_intrinsic(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX_1:%.*]] = phi i32 [ -2147483648, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX:%.*]] = phi i32 [ -2147483648, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[GEP_A]], align 4
; CHECK-NEXT: [[RDX_NEXT]] = call i32 @llvm.smax.i32(i32 [[MAX]], i32 [[TMP2]])
; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP_A_1]], align 4
; CHECK-NEXT: [[RDX_NEXT_1]] = call i32 @llvm.smax.i32(i32 [[MAX_1]], i32 [[TMP3]])
; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EXIT_UNR_LCSSA]]:
; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX_UNR:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[RDX_NEXT]], i32 [[RDX_NEXT_1]])
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[MAX_EPIL_INIT:%.*]] = phi i32 [ -2147483648, %[[ENTRY]] ], [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
; CHECK: [[LOOP_EPIL]]:
; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_EPIL_INIT]]
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 4
; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = call i32 @llvm.smax.i32(i32 [[MAX_EPIL_INIT]], i32 [[TMP4]])
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%max = phi i32 [ -2147483648, %entry ], [ %rdx.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
%1 = load i32, ptr %gep.a, align 4
%rdx.next = call i32 @llvm.smax.i32(i32 %max, i32 %1)
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop, !llvm.loop !0
exit:
%res = phi i32 [ %rdx.next, %loop ]
ret i32 %res
}
; TODO: chained reduction; parallelize this case.
define i32 @test_umin_reduction_intrinsic(ptr %a, i64 %n) {
; CHECK-LABEL: define i32 @test_umin_reduction_intrinsic(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_1:%.*]] = phi i32 [ -1, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN:%.*]] = phi i32 [ -1, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[GEP_A]], align 4
; CHECK-NEXT: [[RDX_NEXT]] = call i32 @llvm.umin.i32(i32 [[MIN]], i32 [[TMP2]])
; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP_A_1]], align 4
; CHECK-NEXT: [[RDX_NEXT_1]] = call i32 @llvm.umin.i32(i32 [[MIN_1]], i32 [[TMP3]])
; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[EXIT_UNR_LCSSA]]:
; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[MIN_UNR:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call i32 @llvm.umin.i32(i32 [[RDX_NEXT]], i32 [[RDX_NEXT_1]])
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[MIN_EPIL_INIT:%.*]] = phi i32 [ -1, %[[ENTRY]] ], [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
; CHECK: [[LOOP_EPIL]]:
; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_EPIL_INIT]]
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 4
; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = call i32 @llvm.umin.i32(i32 [[MIN_EPIL_INIT]], i32 [[TMP4]])
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%min = phi i32 [ -1, %entry ], [ %rdx.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
%1 = load i32, ptr %gep.a, align 4
%rdx.next = call i32 @llvm.umin.i32(i32 %min, i32 %1)
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop, !llvm.loop !0
exit:
%res = phi i32 [ %rdx.next, %loop ]
ret i32 %res
}
; TODO: chained reduction; parallelize this case.
define i32 @test_umax_reduction_intrinsic(ptr %a, i64 %n) {
; CHECK-LABEL: define i32 @test_umax_reduction_intrinsic(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1
; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
; CHECK: [[ENTRY_NEW]]:
; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX_1:%.*]] = phi i32 [ 0, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX:%.*]] = phi i32 [ 0, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[GEP_A]], align 4
; CHECK-NEXT: [[RDX_NEXT]] = call i32 @llvm.umax.i32(i32 [[MAX]], i32 [[TMP2]])
; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_NEXT]]
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP_A_1]], align 4
; CHECK-NEXT: [[RDX_NEXT_1]] = call i32 @llvm.umax.i32(i32 [[MAX_1]], i32 [[TMP3]])
; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2
; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[EXIT_UNR_LCSSA]]:
; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[MAX_UNR:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ]
; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[RDX_NEXT]], i32 [[RDX_NEXT_1]])
; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
; CHECK: [[LOOP_EPIL_PREHEADER]]:
; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[MAX_EPIL_INIT:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ]
; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
; CHECK-NEXT: br label %[[LOOP_EPIL:.*]]
; CHECK: [[LOOP_EPIL]]:
; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV_EPIL_INIT]]
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 4
; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = call i32 @llvm.umax.i32(i32 [[MAX_EPIL_INIT]], i32 [[TMP4]])
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RDX_MINMAX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
; CHECK-NEXT: ret i32 [[RES]]
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%max = phi i32 [ 0, %entry ], [ %rdx.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
%1 = load i32, ptr %gep.a, align 4
%rdx.next = call i32 @llvm.umax.i32(i32 %max, i32 %1)
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop, !llvm.loop !0
exit:
%res = phi i32 [ %rdx.next, %loop ]
ret i32 %res
}
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.unroll.count", i32 2}
!2 = distinct !{!2, !3}
!3 = !{!"llvm.loop.unroll.count", i32 8}