| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6 |
| ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=true -epilogue-vectorization-force-VF=4 -S < %s | FileCheck %s --check-prefixes=CHECK-EPI |
| ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=true -epilogue-vectorization-force-VF=8 -S < %s | FileCheck %s --check-prefixes=CHECK-PARTIAL-RED-EPI |
| |
| target triple = "aarch64" |
| |
| define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 { |
| ; CHECK-EPI-LABEL: define i32 @pred_reduction( |
| ; CHECK-EPI-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; CHECK-EPI-NEXT: [[ITER_CHECK:.*]]: |
| ; CHECK-EPI-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 |
| ; CHECK-EPI-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; CHECK-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; CHECK-EPI-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() |
| ; CHECK-EPI-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4 |
| ; CHECK-EPI-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]] |
| ; CHECK-EPI-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK-EPI: [[VECTOR_PH]]: |
| ; CHECK-EPI-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]] |
| ; CHECK-EPI-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; CHECK-EPI-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK-EPI: [[VECTOR_BODY]]: |
| ; CHECK-EPI-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-EPI-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] |
| ; CHECK-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1 |
| ; CHECK-EPI-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer |
| ; CHECK-EPI-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]] |
| ; CHECK-EPI-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison) |
| ; CHECK-EPI-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32> |
| ; CHECK-EPI-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer |
| ; CHECK-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]]) |
| ; CHECK-EPI-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]] |
| ; CHECK-EPI-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-EPI-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; CHECK-EPI: [[MIDDLE_BLOCK]]: |
| ; CHECK-EPI-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]]) |
| ; CHECK-EPI-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; CHECK-EPI-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; CHECK-EPI: [[VEC_EPILOG_ITER_CHECK]]: |
| ; CHECK-EPI-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4 |
| ; CHECK-EPI-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]] |
| ; CHECK-EPI: [[VEC_EPILOG_PH]]: |
| ; CHECK-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; CHECK-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; CHECK-EPI-NEXT: [[N_MOD_VF2:%.*]] = and i64 [[N]], 3 |
| ; CHECK-EPI-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]] |
| ; CHECK-EPI-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; CHECK-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; CHECK-EPI: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; CHECK-EPI-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; CHECK-EPI-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[TMP10]], %[[VEC_EPILOG_PH]] ], [ [[PREDPHI:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; CHECK-EPI-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX4]] |
| ; CHECK-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP11]], align 1 |
| ; CHECK-EPI-NEXT: [[TMP12:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD6]], zeroinitializer |
| ; CHECK-EPI-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX4]] |
| ; CHECK-EPI-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP13]], <4 x i1> [[TMP12]], <4 x i8> poison) |
| ; CHECK-EPI-NEXT: [[TMP14:%.*]] = zext <4 x i8> [[WIDE_MASKED_LOAD7]] to <4 x i32> |
| ; CHECK-EPI-NEXT: [[TMP15:%.*]] = add <4 x i32> [[VEC_PHI5]], [[TMP14]] |
| ; CHECK-EPI-NEXT: [[PREDPHI]] = select <4 x i1> [[TMP12]], <4 x i32> [[TMP15]], <4 x i32> [[VEC_PHI5]] |
| ; CHECK-EPI-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX4]], 4 |
| ; CHECK-EPI-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]] |
| ; CHECK-EPI-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] |
| ; CHECK-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; CHECK-EPI-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PREDPHI]]) |
| ; CHECK-EPI-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[N]], [[N_VEC3]] |
| ; CHECK-EPI-NEXT: br i1 [[CMP_N9]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; CHECK-EPI: [[VEC_EPILOG_SCALAR_PH]]: |
| ; CHECK-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; CHECK-EPI-NEXT: [[BC_MERGE_RDX9:%.*]] = phi i32 [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; CHECK-EPI-NEXT: br label %[[FOR_BODY:.*]] |
| ; CHECK-EPI: [[FOR_BODY]]: |
| ; CHECK-EPI-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ] |
| ; CHECK-EPI-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX9]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ] |
| ; CHECK-EPI-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]] |
| ; CHECK-EPI-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 |
| ; CHECK-EPI-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0 |
| ; CHECK-EPI-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]] |
| ; CHECK-EPI: [[IF_THEN]]: |
| ; CHECK-EPI-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]] |
| ; CHECK-EPI-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1 |
| ; CHECK-EPI-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32 |
| ; CHECK-EPI-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]] |
| ; CHECK-EPI-NEXT: br label %[[FOR_INC]] |
| ; CHECK-EPI: [[FOR_INC]]: |
| ; CHECK-EPI-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ] |
| ; CHECK-EPI-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 |
| ; CHECK-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] |
| ; CHECK-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] |
| ; CHECK-EPI: [[EXIT]]: |
| ; CHECK-EPI-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; CHECK-EPI-NEXT: ret i32 [[SUM_1_LCSSA]] |
| ; |
| ; CHECK-PARTIAL-RED-EPI-LABEL: define i32 @pred_reduction( |
| ; CHECK-PARTIAL-RED-EPI-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[ITER_CHECK:.*]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 |
| ; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; CHECK-PARTIAL-RED-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4 |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK-PARTIAL-RED-EPI: [[VECTOR_PH]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK-PARTIAL-RED-EPI: [[VECTOR_BODY]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1 |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison) |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32> |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]]) |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; CHECK-PARTIAL-RED-EPI: [[MIDDLE_BLOCK]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]]) |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_ITER_CHECK]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8 |
| ; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]] |
| ; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_PH]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[N_MOD_VF2:%.*]] = and i64 [[N]], 7 |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI5:%.*]] = phi <2 x i32> [ [[TMP10]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX4]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i8>, ptr [[TMP11]], align 1 |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP12:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD6]], zeroinitializer |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX4]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP13]], <8 x i1> [[TMP12]], <8 x i8> poison) |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP14:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD7]] to <8 x i32> |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = select <8 x i1> [[TMP12]], <8 x i32> [[TMP14]], <8 x i32> zeroinitializer |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE8]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI5]], <8 x i32> [[TMP15]]) |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX4]], 8 |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC3]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] |
| ; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE8]]) |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[N]], [[N_VEC3]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[CMP_N10]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_SCALAR_PH]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[FOR_BODY:.*]] |
| ; CHECK-PARTIAL-RED-EPI: [[FOR_BODY]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX10]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0 |
| ; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]] |
| ; CHECK-PARTIAL-RED-EPI: [[IF_THEN]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1 |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32 |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[FOR_INC]] |
| ; CHECK-PARTIAL-RED-EPI: [[FOR_INC]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] |
| ; CHECK-PARTIAL-RED-EPI: [[EXIT]]: |
| ; CHECK-PARTIAL-RED-EPI-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; CHECK-PARTIAL-RED-EPI-NEXT: ret i32 [[SUM_1_LCSSA]] |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] |
| %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ] |
| %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv |
| %c = load i8, ptr %arrayidx, align 1 |
| %tobool.not = icmp eq i8 %c, 0 |
| br i1 %tobool.not, label %for.inc, label %if.then |
| |
| if.then: |
| %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv |
| %val = load i8, ptr %arrayidx2, align 1 |
| %conv = zext i8 %val to i32 |
| %add = add nsw i32 %sum, %conv |
| br label %for.inc |
| |
| for.inc: |
| %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ] |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %exitcond.not = icmp eq i64 %iv.next, %N |
| br i1 %exitcond.not, label %exit, label %for.body |
| |
| exit: |
| ret i32 %sum.1 |
| } |
| |
| attributes #0 = { vscale_range(1,16) "target-features"="+sve" } |
