| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 |
| ; RUN: opt -vectorizer-maximize-bandwidth -mcpu=pwr8 -S -passes=loop-vectorize < %s | FileCheck --check-prefix=P8CHECK %s |
| ; RUN: opt -vectorizer-maximize-bandwidth -mcpu=pwr9 -S -passes=loop-vectorize < %s | FileCheck --check-prefix=P9CHECK %s |
| ; RUN: opt -vectorizer-maximize-bandwidth -mcpu=pwr10 -S -passes=loop-vectorize < %s | FileCheck --check-prefix=P10CHECK %s |
| |
| target datalayout = "e-m:e-Fn32-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512" |
| target triple = "powerpc64le-unknown-linux-gnu" |
| |
| ; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read) uwtable |
| define signext i32 @dotu8(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, i64 noundef %n) { |
| ; P8CHECK-LABEL: define signext i32 @dotu8( |
| ; P8CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; P8CHECK-NEXT: [[ENTRY:.*]]: |
| ; P8CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P8CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P8CHECK: [[ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 |
| ; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P8CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 128 |
| ; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P8CHECK: [[VECTOR_PH]]: |
| ; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 127 |
| ; P8CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; P8CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P8CHECK: [[VECTOR_BODY]]: |
| ; P8CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]] |
| ; P8CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16 |
| ; P8CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32 |
| ; P8CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48 |
| ; P8CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 64 |
| ; P8CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 80 |
| ; P8CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 96 |
| ; P8CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 112 |
| ; P8CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA0:![0-9]+]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <16 x i8>, ptr [[TMP7]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]] |
| ; P8CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16 |
| ; P8CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 32 |
| ; P8CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 48 |
| ; P8CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 64 |
| ; P8CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 80 |
| ; P8CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 96 |
| ; P8CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 112 |
| ; P8CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <16 x i8>, ptr [[TMP12]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <16 x i8>, ptr [[TMP13]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <16 x i8>, ptr [[TMP14]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <16 x i8>, ptr [[TMP15]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[TMP16:%.*]] = zext <16 x i8> [[WIDE_LOAD16]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP17:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <16 x i32> [[TMP16]], [[TMP17]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP18]]) |
| ; P8CHECK-NEXT: [[TMP19:%.*]] = zext <16 x i8> [[WIDE_LOAD17]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP20:%.*]] = zext <16 x i8> [[WIDE_LOAD9]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw <16 x i32> [[TMP19]], [[TMP20]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP21]]) |
| ; P8CHECK-NEXT: [[TMP22:%.*]] = zext <16 x i8> [[WIDE_LOAD18]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP23:%.*]] = zext <16 x i8> [[WIDE_LOAD10]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <16 x i32> [[TMP22]], [[TMP23]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP24]]) |
| ; P8CHECK-NEXT: [[TMP25:%.*]] = zext <16 x i8> [[WIDE_LOAD19]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP26:%.*]] = zext <16 x i8> [[WIDE_LOAD11]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP27:%.*]] = mul nuw nsw <16 x i32> [[TMP25]], [[TMP26]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP27]]) |
| ; P8CHECK-NEXT: [[TMP28:%.*]] = zext <16 x i8> [[WIDE_LOAD20]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP29:%.*]] = zext <16 x i8> [[WIDE_LOAD12]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw <16 x i32> [[TMP28]], [[TMP29]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP30]]) |
| ; P8CHECK-NEXT: [[TMP31:%.*]] = zext <16 x i8> [[WIDE_LOAD21]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP32:%.*]] = zext <16 x i8> [[WIDE_LOAD13]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP33:%.*]] = mul nuw nsw <16 x i32> [[TMP31]], [[TMP32]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP33]]) |
| ; P8CHECK-NEXT: [[TMP34:%.*]] = zext <16 x i8> [[WIDE_LOAD22]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP35:%.*]] = zext <16 x i8> [[WIDE_LOAD14]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP36:%.*]] = mul nuw nsw <16 x i32> [[TMP34]], [[TMP35]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP36]]) |
| ; P8CHECK-NEXT: [[TMP37:%.*]] = zext <16 x i8> [[WIDE_LOAD23]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP38:%.*]] = zext <16 x i8> [[WIDE_LOAD15]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP39:%.*]] = mul nuw nsw <16 x i32> [[TMP37]], [[TMP38]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI8]], <16 x i32> [[TMP39]]) |
| ; P8CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 128 |
| ; P8CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P8CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] |
| ; P8CHECK: [[MIDDLE_BLOCK]]: |
| ; P8CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]] |
| ; P8CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]] |
| ; P8CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]] |
| ; P8CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]] |
| ; P8CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]] |
| ; P8CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]] |
| ; P8CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]] |
| ; P8CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]]) |
| ; P8CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P8CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P8CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16 |
| ; P8CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]] |
| ; P8CHECK: [[VEC_EPILOG_PH]]: |
| ; P8CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 15 |
| ; P8CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]] |
| ; P8CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P8CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P8CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX39]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <16 x i8>, ptr [[TMP43]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX39]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <16 x i8>, ptr [[TMP44]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[TMP45:%.*]] = zext <16 x i8> [[WIDE_LOAD42]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP46:%.*]] = zext <16 x i8> [[WIDE_LOAD41]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP47:%.*]] = mul nuw nsw <16 x i32> [[TMP45]], [[TMP46]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI40]], <16 x i32> [[TMP47]]) |
| ; P8CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 16 |
| ; P8CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]] |
| ; P8CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] |
| ; P8CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P8CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]]) |
| ; P8CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]] |
| ; P8CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P8CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P8CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P8CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P8CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P8CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P8CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P8CHECK: [[FOR_COND_CLEANUP]]: |
| ; P8CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P8CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P8CHECK: [[FOR_BODY]]: |
| ; P8CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P8CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]] |
| ; P8CHECK-NEXT: [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP50]] to i32 |
| ; P8CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]] |
| ; P8CHECK-NEXT: [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[CONV2:%.*]] = zext i8 [[TMP51]] to i32 |
| ; P8CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]] |
| ; P8CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[MUL]], [[SUM_08]] |
| ; P8CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P8CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P8CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] |
| ; |
| ; P9CHECK-LABEL: define signext i32 @dotu8( |
| ; P9CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; P9CHECK-NEXT: [[ENTRY:.*]]: |
| ; P9CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P9CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P9CHECK: [[ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 |
| ; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P9CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 128 |
| ; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P9CHECK: [[VECTOR_PH]]: |
| ; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 127 |
| ; P9CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; P9CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P9CHECK: [[VECTOR_BODY]]: |
| ; P9CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]] |
| ; P9CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16 |
| ; P9CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32 |
| ; P9CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48 |
| ; P9CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 64 |
| ; P9CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 80 |
| ; P9CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 96 |
| ; P9CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 112 |
| ; P9CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA0:![0-9]+]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <16 x i8>, ptr [[TMP7]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]] |
| ; P9CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16 |
| ; P9CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 32 |
| ; P9CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 48 |
| ; P9CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 64 |
| ; P9CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 80 |
| ; P9CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 96 |
| ; P9CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 112 |
| ; P9CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <16 x i8>, ptr [[TMP12]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <16 x i8>, ptr [[TMP13]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <16 x i8>, ptr [[TMP14]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <16 x i8>, ptr [[TMP15]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[TMP16:%.*]] = zext <16 x i8> [[WIDE_LOAD16]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP17:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <16 x i32> [[TMP16]], [[TMP17]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP18]]) |
| ; P9CHECK-NEXT: [[TMP19:%.*]] = zext <16 x i8> [[WIDE_LOAD17]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP20:%.*]] = zext <16 x i8> [[WIDE_LOAD9]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw <16 x i32> [[TMP19]], [[TMP20]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP21]]) |
| ; P9CHECK-NEXT: [[TMP22:%.*]] = zext <16 x i8> [[WIDE_LOAD18]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP23:%.*]] = zext <16 x i8> [[WIDE_LOAD10]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <16 x i32> [[TMP22]], [[TMP23]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP24]]) |
| ; P9CHECK-NEXT: [[TMP25:%.*]] = zext <16 x i8> [[WIDE_LOAD19]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP26:%.*]] = zext <16 x i8> [[WIDE_LOAD11]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP27:%.*]] = mul nuw nsw <16 x i32> [[TMP25]], [[TMP26]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP27]]) |
| ; P9CHECK-NEXT: [[TMP28:%.*]] = zext <16 x i8> [[WIDE_LOAD20]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP29:%.*]] = zext <16 x i8> [[WIDE_LOAD12]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw <16 x i32> [[TMP28]], [[TMP29]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP30]]) |
| ; P9CHECK-NEXT: [[TMP31:%.*]] = zext <16 x i8> [[WIDE_LOAD21]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP32:%.*]] = zext <16 x i8> [[WIDE_LOAD13]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP33:%.*]] = mul nuw nsw <16 x i32> [[TMP31]], [[TMP32]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP33]]) |
| ; P9CHECK-NEXT: [[TMP34:%.*]] = zext <16 x i8> [[WIDE_LOAD22]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP35:%.*]] = zext <16 x i8> [[WIDE_LOAD14]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP36:%.*]] = mul nuw nsw <16 x i32> [[TMP34]], [[TMP35]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP36]]) |
| ; P9CHECK-NEXT: [[TMP37:%.*]] = zext <16 x i8> [[WIDE_LOAD23]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP38:%.*]] = zext <16 x i8> [[WIDE_LOAD15]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP39:%.*]] = mul nuw nsw <16 x i32> [[TMP37]], [[TMP38]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI8]], <16 x i32> [[TMP39]]) |
| ; P9CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 128 |
| ; P9CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P9CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] |
| ; P9CHECK: [[MIDDLE_BLOCK]]: |
| ; P9CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]] |
| ; P9CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]] |
| ; P9CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]] |
| ; P9CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]] |
| ; P9CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]] |
| ; P9CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]] |
| ; P9CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]] |
| ; P9CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]]) |
| ; P9CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P9CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P9CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16 |
| ; P9CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]] |
| ; P9CHECK: [[VEC_EPILOG_PH]]: |
| ; P9CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 15 |
| ; P9CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]] |
| ; P9CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P9CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P9CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX39]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <16 x i8>, ptr [[TMP43]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX39]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <16 x i8>, ptr [[TMP44]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[TMP45:%.*]] = zext <16 x i8> [[WIDE_LOAD42]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP46:%.*]] = zext <16 x i8> [[WIDE_LOAD41]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP47:%.*]] = mul nuw nsw <16 x i32> [[TMP45]], [[TMP46]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI40]], <16 x i32> [[TMP47]]) |
| ; P9CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 16 |
| ; P9CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]] |
| ; P9CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] |
| ; P9CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P9CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]]) |
| ; P9CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]] |
| ; P9CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P9CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P9CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P9CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P9CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P9CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P9CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P9CHECK: [[FOR_COND_CLEANUP]]: |
| ; P9CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P9CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P9CHECK: [[FOR_BODY]]: |
| ; P9CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P9CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]] |
| ; P9CHECK-NEXT: [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP50]] to i32 |
| ; P9CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]] |
| ; P9CHECK-NEXT: [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[CONV2:%.*]] = zext i8 [[TMP51]] to i32 |
| ; P9CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]] |
| ; P9CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[MUL]], [[SUM_08]] |
| ; P9CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P9CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P9CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] |
| ; |
| ; P10CHECK-LABEL: define signext i32 @dotu8( |
| ; P10CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; P10CHECK-NEXT: [[ENTRY:.*]]: |
| ; P10CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P10CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P10CHECK: [[ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 |
| ; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P10CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 128 |
| ; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P10CHECK: [[VECTOR_PH]]: |
| ; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 127 |
| ; P10CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; P10CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P10CHECK: [[VECTOR_BODY]]: |
| ; P10CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]] |
| ; P10CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16 |
| ; P10CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32 |
| ; P10CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48 |
| ; P10CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 64 |
| ; P10CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 80 |
| ; P10CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 96 |
| ; P10CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 112 |
| ; P10CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA0:![0-9]+]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <16 x i8>, ptr [[TMP7]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]] |
| ; P10CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16 |
| ; P10CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 32 |
| ; P10CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 48 |
| ; P10CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 64 |
| ; P10CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 80 |
| ; P10CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 96 |
| ; P10CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 112 |
| ; P10CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <16 x i8>, ptr [[TMP12]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <16 x i8>, ptr [[TMP13]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <16 x i8>, ptr [[TMP14]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <16 x i8>, ptr [[TMP15]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[TMP16:%.*]] = zext <16 x i8> [[WIDE_LOAD16]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP17:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <16 x i32> [[TMP16]], [[TMP17]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP18]]) |
| ; P10CHECK-NEXT: [[TMP19:%.*]] = zext <16 x i8> [[WIDE_LOAD17]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP20:%.*]] = zext <16 x i8> [[WIDE_LOAD9]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw <16 x i32> [[TMP19]], [[TMP20]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP21]]) |
| ; P10CHECK-NEXT: [[TMP22:%.*]] = zext <16 x i8> [[WIDE_LOAD18]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP23:%.*]] = zext <16 x i8> [[WIDE_LOAD10]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <16 x i32> [[TMP22]], [[TMP23]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP24]]) |
| ; P10CHECK-NEXT: [[TMP25:%.*]] = zext <16 x i8> [[WIDE_LOAD19]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP26:%.*]] = zext <16 x i8> [[WIDE_LOAD11]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP27:%.*]] = mul nuw nsw <16 x i32> [[TMP25]], [[TMP26]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP27]]) |
| ; P10CHECK-NEXT: [[TMP28:%.*]] = zext <16 x i8> [[WIDE_LOAD20]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP29:%.*]] = zext <16 x i8> [[WIDE_LOAD12]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw <16 x i32> [[TMP28]], [[TMP29]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP30]]) |
| ; P10CHECK-NEXT: [[TMP31:%.*]] = zext <16 x i8> [[WIDE_LOAD21]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP32:%.*]] = zext <16 x i8> [[WIDE_LOAD13]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP33:%.*]] = mul nuw nsw <16 x i32> [[TMP31]], [[TMP32]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP33]]) |
| ; P10CHECK-NEXT: [[TMP34:%.*]] = zext <16 x i8> [[WIDE_LOAD22]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP35:%.*]] = zext <16 x i8> [[WIDE_LOAD14]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP36:%.*]] = mul nuw nsw <16 x i32> [[TMP34]], [[TMP35]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP36]]) |
| ; P10CHECK-NEXT: [[TMP37:%.*]] = zext <16 x i8> [[WIDE_LOAD23]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP38:%.*]] = zext <16 x i8> [[WIDE_LOAD15]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP39:%.*]] = mul nuw nsw <16 x i32> [[TMP37]], [[TMP38]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI8]], <16 x i32> [[TMP39]]) |
| ; P10CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 128 |
| ; P10CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P10CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] |
| ; P10CHECK: [[MIDDLE_BLOCK]]: |
| ; P10CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]] |
| ; P10CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]] |
| ; P10CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]] |
| ; P10CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]] |
| ; P10CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]] |
| ; P10CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]] |
| ; P10CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]] |
| ; P10CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]]) |
| ; P10CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P10CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P10CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 16 |
| ; P10CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]] |
| ; P10CHECK: [[VEC_EPILOG_PH]]: |
| ; P10CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 15 |
| ; P10CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]] |
| ; P10CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P10CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P10CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX39]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <16 x i8>, ptr [[TMP43]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX39]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <16 x i8>, ptr [[TMP44]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[TMP45:%.*]] = zext <16 x i8> [[WIDE_LOAD42]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP46:%.*]] = zext <16 x i8> [[WIDE_LOAD41]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP47:%.*]] = mul nuw nsw <16 x i32> [[TMP45]], [[TMP46]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI40]], <16 x i32> [[TMP47]]) |
| ; P10CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 16 |
| ; P10CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]] |
| ; P10CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] |
| ; P10CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P10CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]]) |
| ; P10CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]] |
| ; P10CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P10CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P10CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P10CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P10CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P10CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P10CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P10CHECK: [[FOR_COND_CLEANUP]]: |
| ; P10CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P10CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P10CHECK: [[FOR_BODY]]: |
| ; P10CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P10CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]] |
| ; P10CHECK-NEXT: [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP50]] to i32 |
| ; P10CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]] |
| ; P10CHECK-NEXT: [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[CONV2:%.*]] = zext i8 [[TMP51]] to i32 |
| ; P10CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]] |
| ; P10CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[MUL]], [[SUM_08]] |
| ; P10CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P10CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P10CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] |
| ; |
| entry: |
| %cmp7 = icmp sgt i64 %n, 0 |
| br i1 %cmp7, label %for.body.preheader, label %for.cond.cleanup |
| |
| for.body.preheader: ; preds = %entry |
| br label %for.body |
| |
| for.cond.cleanup.loopexit: ; preds = %for.body |
| %add.lcssa = phi i32 [ %add, %for.body ] |
| br label %for.cond.cleanup |
| |
| for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry |
| %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa, %for.cond.cleanup.loopexit ] |
| ret i32 %sum.0.lcssa |
| |
| for.body: ; preds = %for.body.preheader, %for.body |
| %i.09 = phi i64 [ %inc, %for.body ], [ 0, %for.body.preheader ] |
| %sum.08 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ] |
| %arrayidx = getelementptr inbounds nuw i8, ptr %a, i64 %i.09 |
| %0 = load i8, ptr %arrayidx, align 1, !tbaa !8 |
| %conv = zext i8 %0 to i32 |
| %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i64 %i.09 |
| %1 = load i8, ptr %arrayidx1, align 1, !tbaa !8 |
| %conv2 = zext i8 %1 to i32 |
| %mul = mul nuw nsw i32 %conv2, %conv |
| %add = add nuw nsw i32 %mul, %sum.08 |
| %inc = add nuw nsw i64 %i.09, 1 |
| %exitcond.not = icmp eq i64 %inc, %n |
| br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body, !llvm.loop !9 |
| } |
| |
| ; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read) uwtable |
| define signext i32 @dots8(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, i64 noundef %n) { |
| ; P8CHECK-LABEL: define signext i32 @dots8( |
| ; P8CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] { |
| ; P8CHECK-NEXT: [[ENTRY:.*]]: |
| ; P8CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P8CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P8CHECK: [[ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 |
| ; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P8CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64 |
| ; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P8CHECK: [[VECTOR_PH]]: |
| ; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63 |
| ; P8CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; P8CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P8CHECK: [[VECTOR_BODY]]: |
| ; P8CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]] |
| ; P8CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16 |
| ; P8CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32 |
| ; P8CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48 |
| ; P8CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP5:%.*]] = sext <16 x i8> [[WIDE_LOAD5]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP6:%.*]] = sext <16 x i8> [[WIDE_LOAD6]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD7]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]] |
| ; P8CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16 |
| ; P8CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 32 |
| ; P8CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 48 |
| ; P8CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[TMP12:%.*]] = sext <16 x i8> [[WIDE_LOAD8]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP13:%.*]] = sext <16 x i8> [[WIDE_LOAD9]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP14:%.*]] = sext <16 x i8> [[WIDE_LOAD10]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP15:%.*]] = sext <16 x i8> [[WIDE_LOAD11]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP16:%.*]] = mul nsw <16 x i32> [[TMP12]], [[TMP4]] |
| ; P8CHECK-NEXT: [[TMP17:%.*]] = mul nsw <16 x i32> [[TMP13]], [[TMP5]] |
| ; P8CHECK-NEXT: [[TMP18:%.*]] = mul nsw <16 x i32> [[TMP14]], [[TMP6]] |
| ; P8CHECK-NEXT: [[TMP19:%.*]] = mul nsw <16 x i32> [[TMP15]], [[TMP7]] |
| ; P8CHECK-NEXT: [[TMP20]] = add <16 x i32> [[TMP16]], [[VEC_PHI]] |
| ; P8CHECK-NEXT: [[TMP21]] = add <16 x i32> [[TMP17]], [[VEC_PHI2]] |
| ; P8CHECK-NEXT: [[TMP22]] = add <16 x i32> [[TMP18]], [[VEC_PHI3]] |
| ; P8CHECK-NEXT: [[TMP23]] = add <16 x i32> [[TMP19]], [[VEC_PHI4]] |
| ; P8CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64 |
| ; P8CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P8CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] |
| ; P8CHECK: [[MIDDLE_BLOCK]]: |
| ; P8CHECK-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[TMP21]], [[TMP20]] |
| ; P8CHECK-NEXT: [[BIN_RDX12:%.*]] = add <16 x i32> [[TMP22]], [[BIN_RDX]] |
| ; P8CHECK-NEXT: [[BIN_RDX13:%.*]] = add <16 x i32> [[TMP23]], [[BIN_RDX12]] |
| ; P8CHECK-NEXT: [[TMP25:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX13]]) |
| ; P8CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P8CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P8CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4 |
| ; P8CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]] |
| ; P8CHECK: [[VEC_EPILOG_PH]]: |
| ; P8CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[N_MOD_VF14:%.*]] = and i64 [[N]], 3 |
| ; P8CHECK-NEXT: [[N_VEC15:%.*]] = sub i64 [[N]], [[N_MOD_VF14]] |
| ; P8CHECK-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P8CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P8CHECK-NEXT: [[INDEX16:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT20:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI17:%.*]] = phi <4 x i32> [ [[TMP37]], %[[VEC_EPILOG_PH]] ], [ [[TMP32:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX16]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <4 x i8>, ptr [[TMP27]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[TMP28:%.*]] = sext <4 x i8> [[WIDE_LOAD18]] to <4 x i32> |
| ; P8CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX16]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <4 x i8>, ptr [[TMP29]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[TMP30:%.*]] = sext <4 x i8> [[WIDE_LOAD19]] to <4 x i32> |
| ; P8CHECK-NEXT: [[TMP31:%.*]] = mul nsw <4 x i32> [[TMP30]], [[TMP28]] |
| ; P8CHECK-NEXT: [[TMP32]] = add <4 x i32> [[TMP31]], [[VEC_PHI17]] |
| ; P8CHECK-NEXT: [[INDEX_NEXT20]] = add nuw i64 [[INDEX16]], 4 |
| ; P8CHECK-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT20]], [[N_VEC15]] |
| ; P8CHECK-NEXT: br i1 [[TMP33]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] |
| ; P8CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P8CHECK-NEXT: [[TMP34:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP32]]) |
| ; P8CHECK-NEXT: [[CMP_N21:%.*]] = icmp eq i64 [[N]], [[N_VEC15]] |
| ; P8CHECK-NEXT: br i1 [[CMP_N21]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P8CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P8CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[BC_MERGE_RDX21:%.*]] = phi i32 [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P8CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P8CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P8CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP25]], %[[MIDDLE_BLOCK]] ], [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P8CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P8CHECK: [[FOR_COND_CLEANUP]]: |
| ; P8CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P8CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P8CHECK: [[FOR_BODY]]: |
| ; P8CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX21]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P8CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]] |
| ; P8CHECK-NEXT: [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[CONV:%.*]] = sext i8 [[TMP35]] to i32 |
| ; P8CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]] |
| ; P8CHECK-NEXT: [[TMP36:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[CONV2:%.*]] = sext i8 [[TMP36]] to i32 |
| ; P8CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]] |
| ; P8CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]] |
| ; P8CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P8CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P8CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] |
| ; |
| ; P9CHECK-LABEL: define signext i32 @dots8( |
| ; P9CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] { |
| ; P9CHECK-NEXT: [[ENTRY:.*]]: |
| ; P9CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P9CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P9CHECK: [[ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 |
| ; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P9CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64 |
| ; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P9CHECK: [[VECTOR_PH]]: |
| ; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63 |
| ; P9CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; P9CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P9CHECK: [[VECTOR_BODY]]: |
| ; P9CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP40:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP41:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP42:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP43:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP44:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP45:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP46:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP47:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]] |
| ; P9CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8 |
| ; P9CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16 |
| ; P9CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 24 |
| ; P9CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32 |
| ; P9CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 40 |
| ; P9CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48 |
| ; P9CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 56 |
| ; P9CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i8>, ptr [[TMP7]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[TMP8:%.*]] = sext <8 x i8> [[WIDE_LOAD]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP9:%.*]] = sext <8 x i8> [[WIDE_LOAD9]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP10:%.*]] = sext <8 x i8> [[WIDE_LOAD10]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP11:%.*]] = sext <8 x i8> [[WIDE_LOAD11]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP12:%.*]] = sext <8 x i8> [[WIDE_LOAD12]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP13:%.*]] = sext <8 x i8> [[WIDE_LOAD13]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP14:%.*]] = sext <8 x i8> [[WIDE_LOAD14]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP15:%.*]] = sext <8 x i8> [[WIDE_LOAD15]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]] |
| ; P9CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 8 |
| ; P9CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 16 |
| ; P9CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 24 |
| ; P9CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 32 |
| ; P9CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 40 |
| ; P9CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 48 |
| ; P9CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 56 |
| ; P9CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i8>, ptr [[TMP16]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i8>, ptr [[TMP17]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i8>, ptr [[TMP18]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i8>, ptr [[TMP19]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i8>, ptr [[TMP20]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i8>, ptr [[TMP21]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i8>, ptr [[TMP22]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i8>, ptr [[TMP23]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[TMP24:%.*]] = sext <8 x i8> [[WIDE_LOAD16]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP25:%.*]] = sext <8 x i8> [[WIDE_LOAD17]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP26:%.*]] = sext <8 x i8> [[WIDE_LOAD18]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP27:%.*]] = sext <8 x i8> [[WIDE_LOAD19]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP28:%.*]] = sext <8 x i8> [[WIDE_LOAD20]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP29:%.*]] = sext <8 x i8> [[WIDE_LOAD21]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP30:%.*]] = sext <8 x i8> [[WIDE_LOAD22]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP31:%.*]] = sext <8 x i8> [[WIDE_LOAD23]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP32:%.*]] = mul nsw <8 x i32> [[TMP24]], [[TMP8]] |
| ; P9CHECK-NEXT: [[TMP33:%.*]] = mul nsw <8 x i32> [[TMP25]], [[TMP9]] |
| ; P9CHECK-NEXT: [[TMP34:%.*]] = mul nsw <8 x i32> [[TMP26]], [[TMP10]] |
| ; P9CHECK-NEXT: [[TMP35:%.*]] = mul nsw <8 x i32> [[TMP27]], [[TMP11]] |
| ; P9CHECK-NEXT: [[TMP36:%.*]] = mul nsw <8 x i32> [[TMP28]], [[TMP12]] |
| ; P9CHECK-NEXT: [[TMP37:%.*]] = mul nsw <8 x i32> [[TMP29]], [[TMP13]] |
| ; P9CHECK-NEXT: [[TMP38:%.*]] = mul nsw <8 x i32> [[TMP30]], [[TMP14]] |
| ; P9CHECK-NEXT: [[TMP39:%.*]] = mul nsw <8 x i32> [[TMP31]], [[TMP15]] |
| ; P9CHECK-NEXT: [[TMP40]] = add <8 x i32> [[TMP32]], [[VEC_PHI]] |
| ; P9CHECK-NEXT: [[TMP41]] = add <8 x i32> [[TMP33]], [[VEC_PHI2]] |
| ; P9CHECK-NEXT: [[TMP42]] = add <8 x i32> [[TMP34]], [[VEC_PHI3]] |
| ; P9CHECK-NEXT: [[TMP43]] = add <8 x i32> [[TMP35]], [[VEC_PHI4]] |
| ; P9CHECK-NEXT: [[TMP44]] = add <8 x i32> [[TMP36]], [[VEC_PHI5]] |
| ; P9CHECK-NEXT: [[TMP45]] = add <8 x i32> [[TMP37]], [[VEC_PHI6]] |
| ; P9CHECK-NEXT: [[TMP46]] = add <8 x i32> [[TMP38]], [[VEC_PHI7]] |
| ; P9CHECK-NEXT: [[TMP47]] = add <8 x i32> [[TMP39]], [[VEC_PHI8]] |
| ; P9CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64 |
| ; P9CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P9CHECK-NEXT: br i1 [[TMP48]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] |
| ; P9CHECK: [[MIDDLE_BLOCK]]: |
| ; P9CHECK-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[TMP41]], [[TMP40]] |
| ; P9CHECK-NEXT: [[BIN_RDX24:%.*]] = add <8 x i32> [[TMP42]], [[BIN_RDX]] |
| ; P9CHECK-NEXT: [[BIN_RDX25:%.*]] = add <8 x i32> [[TMP43]], [[BIN_RDX24]] |
| ; P9CHECK-NEXT: [[BIN_RDX26:%.*]] = add <8 x i32> [[TMP44]], [[BIN_RDX25]] |
| ; P9CHECK-NEXT: [[BIN_RDX27:%.*]] = add <8 x i32> [[TMP45]], [[BIN_RDX26]] |
| ; P9CHECK-NEXT: [[BIN_RDX28:%.*]] = add <8 x i32> [[TMP46]], [[BIN_RDX27]] |
| ; P9CHECK-NEXT: [[BIN_RDX29:%.*]] = add <8 x i32> [[TMP47]], [[BIN_RDX28]] |
| ; P9CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX29]]) |
| ; P9CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P9CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P9CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8 |
| ; P9CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]] |
| ; P9CHECK: [[VEC_EPILOG_PH]]: |
| ; P9CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[N_MOD_VF30:%.*]] = and i64 [[N]], 7 |
| ; P9CHECK-NEXT: [[N_VEC31:%.*]] = sub i64 [[N]], [[N_MOD_VF30]] |
| ; P9CHECK-NEXT: [[TMP61:%.*]] = insertelement <8 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P9CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P9CHECK-NEXT: [[INDEX32:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT36:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI33:%.*]] = phi <8 x i32> [ [[TMP61]], %[[VEC_EPILOG_PH]] ], [ [[TMP56:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX32]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD34:%.*]] = load <8 x i8>, ptr [[TMP51]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[TMP52:%.*]] = sext <8 x i8> [[WIDE_LOAD34]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX32]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD35:%.*]] = load <8 x i8>, ptr [[TMP53]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[TMP54:%.*]] = sext <8 x i8> [[WIDE_LOAD35]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP55:%.*]] = mul nsw <8 x i32> [[TMP54]], [[TMP52]] |
| ; P9CHECK-NEXT: [[TMP56]] = add <8 x i32> [[TMP55]], [[VEC_PHI33]] |
| ; P9CHECK-NEXT: [[INDEX_NEXT36]] = add nuw i64 [[INDEX32]], 8 |
| ; P9CHECK-NEXT: [[TMP57:%.*]] = icmp eq i64 [[INDEX_NEXT36]], [[N_VEC31]] |
| ; P9CHECK-NEXT: br i1 [[TMP57]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] |
| ; P9CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P9CHECK-NEXT: [[TMP58:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP56]]) |
| ; P9CHECK-NEXT: [[CMP_N37:%.*]] = icmp eq i64 [[N]], [[N_VEC31]] |
| ; P9CHECK-NEXT: br i1 [[CMP_N37]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P9CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P9CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC31]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[BC_MERGE_RDX37:%.*]] = phi i32 [ [[TMP58]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P9CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P9CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P9CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP49]], %[[MIDDLE_BLOCK]] ], [ [[TMP58]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P9CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P9CHECK: [[FOR_COND_CLEANUP]]: |
| ; P9CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P9CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P9CHECK: [[FOR_BODY]]: |
| ; P9CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX37]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P9CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]] |
| ; P9CHECK-NEXT: [[TMP59:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[CONV:%.*]] = sext i8 [[TMP59]] to i32 |
| ; P9CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]] |
| ; P9CHECK-NEXT: [[TMP60:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[CONV2:%.*]] = sext i8 [[TMP60]] to i32 |
| ; P9CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]] |
| ; P9CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]] |
| ; P9CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P9CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P9CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] |
| ; |
| ; P10CHECK-LABEL: define signext i32 @dots8( |
| ; P10CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] { |
| ; P10CHECK-NEXT: [[ENTRY:.*]]: |
| ; P10CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P10CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P10CHECK: [[ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 |
| ; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P10CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64 |
| ; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P10CHECK: [[VECTOR_PH]]: |
| ; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63 |
| ; P10CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; P10CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P10CHECK: [[VECTOR_BODY]]: |
| ; P10CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]] |
| ; P10CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16 |
| ; P10CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32 |
| ; P10CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 48 |
| ; P10CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP5:%.*]] = sext <16 x i8> [[WIDE_LOAD5]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP6:%.*]] = sext <16 x i8> [[WIDE_LOAD6]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD7]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]] |
| ; P10CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 16 |
| ; P10CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 32 |
| ; P10CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 48 |
| ; P10CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[TMP12:%.*]] = sext <16 x i8> [[WIDE_LOAD8]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP13:%.*]] = sext <16 x i8> [[WIDE_LOAD9]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP14:%.*]] = sext <16 x i8> [[WIDE_LOAD10]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP15:%.*]] = sext <16 x i8> [[WIDE_LOAD11]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP16:%.*]] = mul nsw <16 x i32> [[TMP12]], [[TMP4]] |
| ; P10CHECK-NEXT: [[TMP17:%.*]] = mul nsw <16 x i32> [[TMP13]], [[TMP5]] |
| ; P10CHECK-NEXT: [[TMP18:%.*]] = mul nsw <16 x i32> [[TMP14]], [[TMP6]] |
| ; P10CHECK-NEXT: [[TMP19:%.*]] = mul nsw <16 x i32> [[TMP15]], [[TMP7]] |
| ; P10CHECK-NEXT: [[TMP20]] = add <16 x i32> [[TMP16]], [[VEC_PHI]] |
| ; P10CHECK-NEXT: [[TMP21]] = add <16 x i32> [[TMP17]], [[VEC_PHI2]] |
| ; P10CHECK-NEXT: [[TMP22]] = add <16 x i32> [[TMP18]], [[VEC_PHI3]] |
| ; P10CHECK-NEXT: [[TMP23]] = add <16 x i32> [[TMP19]], [[VEC_PHI4]] |
| ; P10CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64 |
| ; P10CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P10CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] |
| ; P10CHECK: [[MIDDLE_BLOCK]]: |
| ; P10CHECK-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[TMP21]], [[TMP20]] |
| ; P10CHECK-NEXT: [[BIN_RDX12:%.*]] = add <16 x i32> [[TMP22]], [[BIN_RDX]] |
| ; P10CHECK-NEXT: [[BIN_RDX13:%.*]] = add <16 x i32> [[TMP23]], [[BIN_RDX12]] |
| ; P10CHECK-NEXT: [[TMP25:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX13]]) |
| ; P10CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P10CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P10CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4 |
| ; P10CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]] |
| ; P10CHECK: [[VEC_EPILOG_PH]]: |
| ; P10CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[N_MOD_VF14:%.*]] = and i64 [[N]], 3 |
| ; P10CHECK-NEXT: [[N_VEC15:%.*]] = sub i64 [[N]], [[N_MOD_VF14]] |
| ; P10CHECK-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P10CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P10CHECK-NEXT: [[INDEX16:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT20:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI17:%.*]] = phi <4 x i32> [ [[TMP37]], %[[VEC_EPILOG_PH]] ], [ [[TMP32:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX16]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <4 x i8>, ptr [[TMP27]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[TMP28:%.*]] = sext <4 x i8> [[WIDE_LOAD18]] to <4 x i32> |
| ; P10CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX16]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <4 x i8>, ptr [[TMP29]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[TMP30:%.*]] = sext <4 x i8> [[WIDE_LOAD19]] to <4 x i32> |
| ; P10CHECK-NEXT: [[TMP31:%.*]] = mul nsw <4 x i32> [[TMP30]], [[TMP28]] |
| ; P10CHECK-NEXT: [[TMP32]] = add <4 x i32> [[TMP31]], [[VEC_PHI17]] |
| ; P10CHECK-NEXT: [[INDEX_NEXT20]] = add nuw i64 [[INDEX16]], 4 |
| ; P10CHECK-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT20]], [[N_VEC15]] |
| ; P10CHECK-NEXT: br i1 [[TMP33]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] |
| ; P10CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P10CHECK-NEXT: [[TMP34:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP32]]) |
| ; P10CHECK-NEXT: [[CMP_N21:%.*]] = icmp eq i64 [[N]], [[N_VEC15]] |
| ; P10CHECK-NEXT: br i1 [[CMP_N21]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P10CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P10CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[BC_MERGE_RDX21:%.*]] = phi i32 [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP25]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P10CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P10CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P10CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP25]], %[[MIDDLE_BLOCK]] ], [ [[TMP34]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P10CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P10CHECK: [[FOR_COND_CLEANUP]]: |
| ; P10CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P10CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P10CHECK: [[FOR_BODY]]: |
| ; P10CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX21]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P10CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]] |
| ; P10CHECK-NEXT: [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[CONV:%.*]] = sext i8 [[TMP35]] to i32 |
| ; P10CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]] |
| ; P10CHECK-NEXT: [[TMP36:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[CONV2:%.*]] = sext i8 [[TMP36]] to i32 |
| ; P10CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]] |
| ; P10CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]] |
| ; P10CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P10CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P10CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] |
| ; |
| entry: |
| %cmp7 = icmp sgt i64 %n, 0 |
| br i1 %cmp7, label %for.body, label %for.cond.cleanup |
| |
| for.cond.cleanup: ; preds = %for.body, %entry |
| %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ] |
| ret i32 %sum.0.lcssa |
| |
| for.body: ; preds = %entry, %for.body |
| %i.09 = phi i64 [ %inc, %for.body ], [ 0, %entry ] |
| %sum.08 = phi i32 [ %add, %for.body ], [ 0, %entry ] |
| %arrayidx = getelementptr inbounds nuw i8, ptr %a, i64 %i.09 |
| %0 = load i8, ptr %arrayidx, align 1, !tbaa !8 |
| %conv = sext i8 %0 to i32 |
| %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i64 %i.09 |
| %1 = load i8, ptr %arrayidx1, align 1, !tbaa !8 |
| %conv2 = sext i8 %1 to i32 |
| %mul = mul nsw i32 %conv2, %conv |
| %add = add nsw i32 %mul, %sum.08 |
| %inc = add nuw nsw i64 %i.09, 1 |
| %exitcond.not = icmp eq i64 %inc, %n |
| br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !11 |
| } |
| |
| ; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read) uwtable |
| define signext i32 @dotus8(ptr nofree noundef readonly captures(none) %a, ptr nofree noundef readonly captures(none) %b, i64 noundef %n) { |
| ; P8CHECK-LABEL: define signext i32 @dotus8( |
| ; P8CHECK-SAME: ptr nofree noundef readonly captures(none) [[A:%.*]], ptr nofree noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] { |
| ; P8CHECK-NEXT: [[ENTRY:.*]]: |
| ; P8CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P8CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P8CHECK: [[ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 |
| ; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P8CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 128 |
| ; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P8CHECK: [[VECTOR_PH]]: |
| ; P8CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 127 |
| ; P8CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] |
| ; P8CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P8CHECK: [[VECTOR_BODY]]: |
| ; P8CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]] |
| ; P8CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16 |
| ; P8CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32 |
| ; P8CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 48 |
| ; P8CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 64 |
| ; P8CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80 |
| ; P8CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 96 |
| ; P8CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 112 |
| ; P8CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <16 x i8>, ptr [[TMP7]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]] |
| ; P8CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 16 |
| ; P8CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 32 |
| ; P8CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 48 |
| ; P8CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 64 |
| ; P8CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 80 |
| ; P8CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 96 |
| ; P8CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 112 |
| ; P8CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <16 x i8>, ptr [[TMP12]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <16 x i8>, ptr [[TMP13]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <16 x i8>, ptr [[TMP14]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <16 x i8>, ptr [[TMP15]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <16 x i8>, ptr [[TMP16]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[TMP17:%.*]] = sext <16 x i8> [[WIDE_LOAD16]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP18:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP19:%.*]] = mul nsw <16 x i32> [[TMP17]], [[TMP18]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP19]]) |
| ; P8CHECK-NEXT: [[TMP20:%.*]] = sext <16 x i8> [[WIDE_LOAD17]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP21:%.*]] = zext <16 x i8> [[WIDE_LOAD9]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP22:%.*]] = mul nsw <16 x i32> [[TMP20]], [[TMP21]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP22]]) |
| ; P8CHECK-NEXT: [[TMP23:%.*]] = sext <16 x i8> [[WIDE_LOAD18]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP24:%.*]] = zext <16 x i8> [[WIDE_LOAD10]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP25:%.*]] = mul nsw <16 x i32> [[TMP23]], [[TMP24]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP25]]) |
| ; P8CHECK-NEXT: [[TMP26:%.*]] = sext <16 x i8> [[WIDE_LOAD19]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP27:%.*]] = zext <16 x i8> [[WIDE_LOAD11]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP28:%.*]] = mul nsw <16 x i32> [[TMP26]], [[TMP27]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP28]]) |
| ; P8CHECK-NEXT: [[TMP29:%.*]] = sext <16 x i8> [[WIDE_LOAD20]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP30:%.*]] = zext <16 x i8> [[WIDE_LOAD12]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP31:%.*]] = mul nsw <16 x i32> [[TMP29]], [[TMP30]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP31]]) |
| ; P8CHECK-NEXT: [[TMP32:%.*]] = sext <16 x i8> [[WIDE_LOAD21]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP33:%.*]] = zext <16 x i8> [[WIDE_LOAD13]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP34:%.*]] = mul nsw <16 x i32> [[TMP32]], [[TMP33]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP34]]) |
| ; P8CHECK-NEXT: [[TMP35:%.*]] = sext <16 x i8> [[WIDE_LOAD22]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP36:%.*]] = zext <16 x i8> [[WIDE_LOAD14]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP37:%.*]] = mul nsw <16 x i32> [[TMP35]], [[TMP36]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP37]]) |
| ; P8CHECK-NEXT: [[TMP38:%.*]] = sext <16 x i8> [[WIDE_LOAD23]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP39:%.*]] = zext <16 x i8> [[WIDE_LOAD15]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP40:%.*]] = mul nsw <16 x i32> [[TMP38]], [[TMP39]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI8]], <16 x i32> [[TMP40]]) |
| ; P8CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 128 |
| ; P8CHECK-NEXT: [[TMP41:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P8CHECK-NEXT: br i1 [[TMP41]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] |
| ; P8CHECK: [[MIDDLE_BLOCK]]: |
| ; P8CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]] |
| ; P8CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]] |
| ; P8CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]] |
| ; P8CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]] |
| ; P8CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]] |
| ; P8CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]] |
| ; P8CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]] |
| ; P8CHECK-NEXT: [[TMP42:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]]) |
| ; P8CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P8CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P8CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16 |
| ; P8CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7]] |
| ; P8CHECK: [[VEC_EPILOG_PH]]: |
| ; P8CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP42]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[TMP43:%.*]] = and i64 [[N]], 15 |
| ; P8CHECK-NEXT: [[N_VEC37:%.*]] = sub i64 [[N]], [[TMP43]] |
| ; P8CHECK-NEXT: [[TMP44:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P8CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P8CHECK-NEXT: [[INDEX38:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI39:%.*]] = phi <4 x i32> [ [[TMP44]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE42:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX38]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD40:%.*]] = load <16 x i8>, ptr [[TMP45]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX38]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <16 x i8>, ptr [[TMP46]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[TMP47:%.*]] = sext <16 x i8> [[WIDE_LOAD41]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP48:%.*]] = zext <16 x i8> [[WIDE_LOAD40]] to <16 x i32> |
| ; P8CHECK-NEXT: [[TMP49:%.*]] = mul nsw <16 x i32> [[TMP47]], [[TMP48]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE42]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI39]], <16 x i32> [[TMP49]]) |
| ; P8CHECK-NEXT: [[INDEX_NEXT43]] = add nuw i64 [[INDEX38]], 16 |
| ; P8CHECK-NEXT: [[TMP50:%.*]] = icmp eq i64 [[INDEX_NEXT43]], [[N_VEC37]] |
| ; P8CHECK-NEXT: br i1 [[TMP50]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] |
| ; P8CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P8CHECK-NEXT: [[TMP51:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE42]]) |
| ; P8CHECK-NEXT: [[CMP_N44:%.*]] = icmp eq i64 [[N]], [[N_VEC37]] |
| ; P8CHECK-NEXT: br i1 [[CMP_N44]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P8CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P8CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC37]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP51]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP42]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P8CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P8CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P8CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP42]], %[[MIDDLE_BLOCK]] ], [ [[TMP51]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P8CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P8CHECK: [[FOR_COND_CLEANUP]]: |
| ; P8CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P8CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P8CHECK: [[FOR_BODY]]: |
| ; P8CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P8CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]] |
| ; P8CHECK-NEXT: [[TMP52:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP52]] to i32 |
| ; P8CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]] |
| ; P8CHECK-NEXT: [[TMP53:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P8CHECK-NEXT: [[CONV2:%.*]] = sext i8 [[TMP53]] to i32 |
| ; P8CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]] |
| ; P8CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]] |
| ; P8CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P8CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P8CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] |
| ; |
| ; P9CHECK-LABEL: define signext i32 @dotus8( |
| ; P9CHECK-SAME: ptr nofree noundef readonly captures(none) [[A:%.*]], ptr nofree noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] { |
| ; P9CHECK-NEXT: [[ENTRY:.*]]: |
| ; P9CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P9CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P9CHECK: [[ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 |
| ; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P9CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 128 |
| ; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P9CHECK: [[VECTOR_PH]]: |
| ; P9CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 127 |
| ; P9CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] |
| ; P9CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P9CHECK: [[VECTOR_BODY]]: |
| ; P9CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]] |
| ; P9CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16 |
| ; P9CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32 |
| ; P9CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 48 |
| ; P9CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 64 |
| ; P9CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80 |
| ; P9CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 96 |
| ; P9CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 112 |
| ; P9CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <16 x i8>, ptr [[TMP7]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]] |
| ; P9CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 16 |
| ; P9CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 32 |
| ; P9CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 48 |
| ; P9CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 64 |
| ; P9CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 80 |
| ; P9CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 96 |
| ; P9CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 112 |
| ; P9CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <16 x i8>, ptr [[TMP12]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <16 x i8>, ptr [[TMP13]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <16 x i8>, ptr [[TMP14]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <16 x i8>, ptr [[TMP15]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <16 x i8>, ptr [[TMP16]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[TMP17:%.*]] = sext <16 x i8> [[WIDE_LOAD16]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP18:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP19:%.*]] = mul nsw <16 x i32> [[TMP17]], [[TMP18]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP19]]) |
| ; P9CHECK-NEXT: [[TMP20:%.*]] = sext <16 x i8> [[WIDE_LOAD17]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP21:%.*]] = zext <16 x i8> [[WIDE_LOAD9]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP22:%.*]] = mul nsw <16 x i32> [[TMP20]], [[TMP21]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP22]]) |
| ; P9CHECK-NEXT: [[TMP23:%.*]] = sext <16 x i8> [[WIDE_LOAD18]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP24:%.*]] = zext <16 x i8> [[WIDE_LOAD10]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP25:%.*]] = mul nsw <16 x i32> [[TMP23]], [[TMP24]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP25]]) |
| ; P9CHECK-NEXT: [[TMP26:%.*]] = sext <16 x i8> [[WIDE_LOAD19]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP27:%.*]] = zext <16 x i8> [[WIDE_LOAD11]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP28:%.*]] = mul nsw <16 x i32> [[TMP26]], [[TMP27]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP28]]) |
| ; P9CHECK-NEXT: [[TMP29:%.*]] = sext <16 x i8> [[WIDE_LOAD20]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP30:%.*]] = zext <16 x i8> [[WIDE_LOAD12]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP31:%.*]] = mul nsw <16 x i32> [[TMP29]], [[TMP30]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP31]]) |
| ; P9CHECK-NEXT: [[TMP32:%.*]] = sext <16 x i8> [[WIDE_LOAD21]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP33:%.*]] = zext <16 x i8> [[WIDE_LOAD13]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP34:%.*]] = mul nsw <16 x i32> [[TMP32]], [[TMP33]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP34]]) |
| ; P9CHECK-NEXT: [[TMP35:%.*]] = sext <16 x i8> [[WIDE_LOAD22]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP36:%.*]] = zext <16 x i8> [[WIDE_LOAD14]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP37:%.*]] = mul nsw <16 x i32> [[TMP35]], [[TMP36]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP37]]) |
| ; P9CHECK-NEXT: [[TMP38:%.*]] = sext <16 x i8> [[WIDE_LOAD23]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP39:%.*]] = zext <16 x i8> [[WIDE_LOAD15]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP40:%.*]] = mul nsw <16 x i32> [[TMP38]], [[TMP39]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI8]], <16 x i32> [[TMP40]]) |
| ; P9CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 128 |
| ; P9CHECK-NEXT: [[TMP41:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P9CHECK-NEXT: br i1 [[TMP41]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] |
| ; P9CHECK: [[MIDDLE_BLOCK]]: |
| ; P9CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]] |
| ; P9CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]] |
| ; P9CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]] |
| ; P9CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]] |
| ; P9CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]] |
| ; P9CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]] |
| ; P9CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]] |
| ; P9CHECK-NEXT: [[TMP42:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]]) |
| ; P9CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P9CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P9CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16 |
| ; P9CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7]] |
| ; P9CHECK: [[VEC_EPILOG_PH]]: |
| ; P9CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP42]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[TMP43:%.*]] = and i64 [[N]], 15 |
| ; P9CHECK-NEXT: [[N_VEC37:%.*]] = sub i64 [[N]], [[TMP43]] |
| ; P9CHECK-NEXT: [[TMP44:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P9CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P9CHECK-NEXT: [[INDEX38:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI39:%.*]] = phi <4 x i32> [ [[TMP44]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE42:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX38]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD40:%.*]] = load <16 x i8>, ptr [[TMP45]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX38]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <16 x i8>, ptr [[TMP46]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[TMP47:%.*]] = sext <16 x i8> [[WIDE_LOAD41]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP48:%.*]] = zext <16 x i8> [[WIDE_LOAD40]] to <16 x i32> |
| ; P9CHECK-NEXT: [[TMP49:%.*]] = mul nsw <16 x i32> [[TMP47]], [[TMP48]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE42]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI39]], <16 x i32> [[TMP49]]) |
| ; P9CHECK-NEXT: [[INDEX_NEXT43]] = add nuw i64 [[INDEX38]], 16 |
| ; P9CHECK-NEXT: [[TMP50:%.*]] = icmp eq i64 [[INDEX_NEXT43]], [[N_VEC37]] |
| ; P9CHECK-NEXT: br i1 [[TMP50]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] |
| ; P9CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P9CHECK-NEXT: [[TMP51:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE42]]) |
| ; P9CHECK-NEXT: [[CMP_N44:%.*]] = icmp eq i64 [[N]], [[N_VEC37]] |
| ; P9CHECK-NEXT: br i1 [[CMP_N44]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P9CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P9CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC37]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP51]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP42]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P9CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P9CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P9CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP42]], %[[MIDDLE_BLOCK]] ], [ [[TMP51]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P9CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P9CHECK: [[FOR_COND_CLEANUP]]: |
| ; P9CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P9CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P9CHECK: [[FOR_BODY]]: |
| ; P9CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P9CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]] |
| ; P9CHECK-NEXT: [[TMP52:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP52]] to i32 |
| ; P9CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]] |
| ; P9CHECK-NEXT: [[TMP53:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P9CHECK-NEXT: [[CONV2:%.*]] = sext i8 [[TMP53]] to i32 |
| ; P9CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]] |
| ; P9CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]] |
| ; P9CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P9CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P9CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] |
| ; |
| ; P10CHECK-LABEL: define signext i32 @dotus8( |
| ; P10CHECK-SAME: ptr nofree noundef readonly captures(none) [[A:%.*]], ptr nofree noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] { |
| ; P10CHECK-NEXT: [[ENTRY:.*]]: |
| ; P10CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P10CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P10CHECK: [[ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 |
| ; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P10CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 128 |
| ; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P10CHECK: [[VECTOR_PH]]: |
| ; P10CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 127 |
| ; P10CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] |
| ; P10CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P10CHECK: [[VECTOR_BODY]]: |
| ; P10CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]] |
| ; P10CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16 |
| ; P10CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32 |
| ; P10CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 48 |
| ; P10CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 64 |
| ; P10CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80 |
| ; P10CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 96 |
| ; P10CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 112 |
| ; P10CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <16 x i8>, ptr [[TMP7]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <16 x i8>, ptr [[TMP8]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]] |
| ; P10CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 16 |
| ; P10CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 32 |
| ; P10CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 48 |
| ; P10CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 64 |
| ; P10CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 80 |
| ; P10CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 96 |
| ; P10CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 112 |
| ; P10CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <16 x i8>, ptr [[TMP10]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <16 x i8>, ptr [[TMP12]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <16 x i8>, ptr [[TMP13]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <16 x i8>, ptr [[TMP14]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <16 x i8>, ptr [[TMP15]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <16 x i8>, ptr [[TMP16]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[TMP17:%.*]] = sext <16 x i8> [[WIDE_LOAD16]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP18:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP19:%.*]] = mul nsw <16 x i32> [[TMP17]], [[TMP18]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP19]]) |
| ; P10CHECK-NEXT: [[TMP20:%.*]] = sext <16 x i8> [[WIDE_LOAD17]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP21:%.*]] = zext <16 x i8> [[WIDE_LOAD9]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP22:%.*]] = mul nsw <16 x i32> [[TMP20]], [[TMP21]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP22]]) |
| ; P10CHECK-NEXT: [[TMP23:%.*]] = sext <16 x i8> [[WIDE_LOAD18]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP24:%.*]] = zext <16 x i8> [[WIDE_LOAD10]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP25:%.*]] = mul nsw <16 x i32> [[TMP23]], [[TMP24]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP25]]) |
| ; P10CHECK-NEXT: [[TMP26:%.*]] = sext <16 x i8> [[WIDE_LOAD19]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP27:%.*]] = zext <16 x i8> [[WIDE_LOAD11]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP28:%.*]] = mul nsw <16 x i32> [[TMP26]], [[TMP27]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP28]]) |
| ; P10CHECK-NEXT: [[TMP29:%.*]] = sext <16 x i8> [[WIDE_LOAD20]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP30:%.*]] = zext <16 x i8> [[WIDE_LOAD12]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP31:%.*]] = mul nsw <16 x i32> [[TMP29]], [[TMP30]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP31]]) |
| ; P10CHECK-NEXT: [[TMP32:%.*]] = sext <16 x i8> [[WIDE_LOAD21]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP33:%.*]] = zext <16 x i8> [[WIDE_LOAD13]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP34:%.*]] = mul nsw <16 x i32> [[TMP32]], [[TMP33]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP34]]) |
| ; P10CHECK-NEXT: [[TMP35:%.*]] = sext <16 x i8> [[WIDE_LOAD22]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP36:%.*]] = zext <16 x i8> [[WIDE_LOAD14]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP37:%.*]] = mul nsw <16 x i32> [[TMP35]], [[TMP36]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP37]]) |
| ; P10CHECK-NEXT: [[TMP38:%.*]] = sext <16 x i8> [[WIDE_LOAD23]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP39:%.*]] = zext <16 x i8> [[WIDE_LOAD15]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP40:%.*]] = mul nsw <16 x i32> [[TMP38]], [[TMP39]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI8]], <16 x i32> [[TMP40]]) |
| ; P10CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 128 |
| ; P10CHECK-NEXT: [[TMP41:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P10CHECK-NEXT: br i1 [[TMP41]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] |
| ; P10CHECK: [[MIDDLE_BLOCK]]: |
| ; P10CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]] |
| ; P10CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]] |
| ; P10CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]] |
| ; P10CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]] |
| ; P10CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]] |
| ; P10CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]] |
| ; P10CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]] |
| ; P10CHECK-NEXT: [[TMP42:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]]) |
| ; P10CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P10CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P10CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16 |
| ; P10CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7]] |
| ; P10CHECK: [[VEC_EPILOG_PH]]: |
| ; P10CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP42]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[TMP43:%.*]] = and i64 [[N]], 15 |
| ; P10CHECK-NEXT: [[N_VEC37:%.*]] = sub i64 [[N]], [[TMP43]] |
| ; P10CHECK-NEXT: [[TMP44:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P10CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P10CHECK-NEXT: [[INDEX38:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI39:%.*]] = phi <4 x i32> [ [[TMP44]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE42:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[TMP45:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX38]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD40:%.*]] = load <16 x i8>, ptr [[TMP45]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX38]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <16 x i8>, ptr [[TMP46]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[TMP47:%.*]] = sext <16 x i8> [[WIDE_LOAD41]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP48:%.*]] = zext <16 x i8> [[WIDE_LOAD40]] to <16 x i32> |
| ; P10CHECK-NEXT: [[TMP49:%.*]] = mul nsw <16 x i32> [[TMP47]], [[TMP48]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE42]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI39]], <16 x i32> [[TMP49]]) |
| ; P10CHECK-NEXT: [[INDEX_NEXT43]] = add nuw i64 [[INDEX38]], 16 |
| ; P10CHECK-NEXT: [[TMP50:%.*]] = icmp eq i64 [[INDEX_NEXT43]], [[N_VEC37]] |
| ; P10CHECK-NEXT: br i1 [[TMP50]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] |
| ; P10CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P10CHECK-NEXT: [[TMP51:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE42]]) |
| ; P10CHECK-NEXT: [[CMP_N44:%.*]] = icmp eq i64 [[N]], [[N_VEC37]] |
| ; P10CHECK-NEXT: br i1 [[CMP_N44]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P10CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P10CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC37]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP51]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP42]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P10CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P10CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P10CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP42]], %[[MIDDLE_BLOCK]] ], [ [[TMP51]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P10CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P10CHECK: [[FOR_COND_CLEANUP]]: |
| ; P10CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P10CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P10CHECK: [[FOR_BODY]]: |
| ; P10CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P10CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[I_09]] |
| ; P10CHECK-NEXT: [[TMP52:%.*]] = load i8, ptr [[ARRAYIDX]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP52]] to i32 |
| ; P10CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[I_09]] |
| ; P10CHECK-NEXT: [[TMP53:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1, !tbaa [[CHAR_TBAA0]] |
| ; P10CHECK-NEXT: [[CONV2:%.*]] = sext i8 [[TMP53]] to i32 |
| ; P10CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]] |
| ; P10CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]] |
| ; P10CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P10CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P10CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] |
| ; |
| entry: |
| %cmp7 = icmp sgt i64 %n, 0 |
| br i1 %cmp7, label %for.body, label %for.cond.cleanup |
| |
| for.cond.cleanup: ; preds = %for.body, %entry |
| %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ] |
| ret i32 %sum.0.lcssa |
| |
| for.body: ; preds = %entry, %for.body |
| %i.09 = phi i64 [ %inc, %for.body ], [ 0, %entry ] |
| %sum.08 = phi i32 [ %add, %for.body ], [ 0, %entry ] |
| %arrayidx = getelementptr inbounds nuw i8, ptr %a, i64 %i.09 |
| %0 = load i8, ptr %arrayidx, align 1, !tbaa !8 |
| %conv = zext i8 %0 to i32 |
| %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i64 %i.09 |
| %1 = load i8, ptr %arrayidx1, align 1, !tbaa !8 |
| %conv2 = sext i8 %1 to i32 |
| %mul = mul nsw i32 %conv2, %conv |
| %add = add nsw i32 %mul, %sum.08 |
| %inc = add nuw nsw i64 %i.09, 1 |
| %exitcond.not = icmp eq i64 %inc, %n |
| br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !13 |
| } |
| |
| ; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read) uwtable |
| define signext range(i32 0, -2147483648) i32 @dotu16(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, i64 noundef %n) { |
| ; P8CHECK-LABEL: define signext range(i32 0, -2147483648) i32 @dotu16( |
| ; P8CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] { |
| ; P8CHECK-NEXT: [[ENTRY:.*]]: |
| ; P8CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P8CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P8CHECK: [[ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 |
| ; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P8CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64 |
| ; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P8CHECK: [[VECTOR_PH]]: |
| ; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63 |
| ; P8CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; P8CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P8CHECK: [[VECTOR_BODY]]: |
| ; P8CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX]] |
| ; P8CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 8 |
| ; P8CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 16 |
| ; P8CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 24 |
| ; P8CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 32 |
| ; P8CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 40 |
| ; P8CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 48 |
| ; P8CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 56 |
| ; P8CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2, !tbaa [[SHORT_TBAA17:![0-9]+]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i16>, ptr [[TMP6]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[TMP7]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX]] |
| ; P8CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 8 |
| ; P8CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 16 |
| ; P8CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 24 |
| ; P8CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 32 |
| ; P8CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 40 |
| ; P8CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 48 |
| ; P8CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 56 |
| ; P8CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i16>, ptr [[TMP8]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[TMP9]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i16>, ptr [[TMP12]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i16>, ptr [[TMP13]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i16>, ptr [[TMP14]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i16>, ptr [[TMP15]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[TMP16:%.*]] = zext <8 x i16> [[WIDE_LOAD16]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP17:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <8 x i32> [[TMP16]], [[TMP17]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI]], <8 x i32> [[TMP18]]) |
| ; P8CHECK-NEXT: [[TMP19:%.*]] = zext <8 x i16> [[WIDE_LOAD17]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP20:%.*]] = zext <8 x i16> [[WIDE_LOAD9]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw <8 x i32> [[TMP19]], [[TMP20]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI2]], <8 x i32> [[TMP21]]) |
| ; P8CHECK-NEXT: [[TMP22:%.*]] = zext <8 x i16> [[WIDE_LOAD18]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP23:%.*]] = zext <8 x i16> [[WIDE_LOAD10]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <8 x i32> [[TMP22]], [[TMP23]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI3]], <8 x i32> [[TMP24]]) |
| ; P8CHECK-NEXT: [[TMP25:%.*]] = zext <8 x i16> [[WIDE_LOAD19]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP26:%.*]] = zext <8 x i16> [[WIDE_LOAD11]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP27:%.*]] = mul nuw nsw <8 x i32> [[TMP25]], [[TMP26]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI4]], <8 x i32> [[TMP27]]) |
| ; P8CHECK-NEXT: [[TMP28:%.*]] = zext <8 x i16> [[WIDE_LOAD20]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP29:%.*]] = zext <8 x i16> [[WIDE_LOAD12]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw <8 x i32> [[TMP28]], [[TMP29]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI5]], <8 x i32> [[TMP30]]) |
| ; P8CHECK-NEXT: [[TMP31:%.*]] = zext <8 x i16> [[WIDE_LOAD21]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP32:%.*]] = zext <8 x i16> [[WIDE_LOAD13]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP33:%.*]] = mul nuw nsw <8 x i32> [[TMP31]], [[TMP32]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI6]], <8 x i32> [[TMP33]]) |
| ; P8CHECK-NEXT: [[TMP34:%.*]] = zext <8 x i16> [[WIDE_LOAD22]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP35:%.*]] = zext <8 x i16> [[WIDE_LOAD14]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP36:%.*]] = mul nuw nsw <8 x i32> [[TMP34]], [[TMP35]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI7]], <8 x i32> [[TMP36]]) |
| ; P8CHECK-NEXT: [[TMP37:%.*]] = zext <8 x i16> [[WIDE_LOAD23]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP38:%.*]] = zext <8 x i16> [[WIDE_LOAD15]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP39:%.*]] = mul nuw nsw <8 x i32> [[TMP37]], [[TMP38]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI8]], <8 x i32> [[TMP39]]) |
| ; P8CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64 |
| ; P8CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P8CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] |
| ; P8CHECK: [[MIDDLE_BLOCK]]: |
| ; P8CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]] |
| ; P8CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]] |
| ; P8CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]] |
| ; P8CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]] |
| ; P8CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]] |
| ; P8CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]] |
| ; P8CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]] |
| ; P8CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]]) |
| ; P8CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P8CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P8CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8 |
| ; P8CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF20:![0-9]+]] |
| ; P8CHECK: [[VEC_EPILOG_PH]]: |
| ; P8CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7 |
| ; P8CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]] |
| ; P8CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P8CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P8CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <8 x i16>, ptr [[TMP44]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[TMP45:%.*]] = zext <8 x i16> [[WIDE_LOAD42]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP46:%.*]] = zext <8 x i16> [[WIDE_LOAD41]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP47:%.*]] = mul nuw nsw <8 x i32> [[TMP45]], [[TMP46]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI40]], <8 x i32> [[TMP47]]) |
| ; P8CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 8 |
| ; P8CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]] |
| ; P8CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] |
| ; P8CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P8CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]]) |
| ; P8CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]] |
| ; P8CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P8CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P8CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P8CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P8CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P8CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P8CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P8CHECK: [[FOR_COND_CLEANUP]]: |
| ; P8CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P8CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P8CHECK: [[FOR_BODY]]: |
| ; P8CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P8CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]] |
| ; P8CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[CONV:%.*]] = zext i16 [[TMP50]] to i32 |
| ; P8CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[I_09]] |
| ; P8CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[CONV2:%.*]] = zext i16 [[TMP51]] to i32 |
| ; P8CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]] |
| ; P8CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[MUL]], [[SUM_08]] |
| ; P8CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P8CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P8CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] |
| ; |
| ; P9CHECK-LABEL: define signext range(i32 0, -2147483648) i32 @dotu16( |
| ; P9CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] { |
| ; P9CHECK-NEXT: [[ENTRY:.*]]: |
| ; P9CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P9CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P9CHECK: [[ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 |
| ; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P9CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64 |
| ; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P9CHECK: [[VECTOR_PH]]: |
| ; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63 |
| ; P9CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; P9CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P9CHECK: [[VECTOR_BODY]]: |
| ; P9CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX]] |
| ; P9CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 8 |
| ; P9CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 16 |
| ; P9CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 24 |
| ; P9CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 32 |
| ; P9CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 40 |
| ; P9CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 48 |
| ; P9CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 56 |
| ; P9CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2, !tbaa [[SHORT_TBAA17:![0-9]+]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i16>, ptr [[TMP6]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[TMP7]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX]] |
| ; P9CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 8 |
| ; P9CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 16 |
| ; P9CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 24 |
| ; P9CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 32 |
| ; P9CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 40 |
| ; P9CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 48 |
| ; P9CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 56 |
| ; P9CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i16>, ptr [[TMP8]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[TMP9]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i16>, ptr [[TMP12]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i16>, ptr [[TMP13]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i16>, ptr [[TMP14]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i16>, ptr [[TMP15]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[TMP16:%.*]] = zext <8 x i16> [[WIDE_LOAD16]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP17:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <8 x i32> [[TMP16]], [[TMP17]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI]], <8 x i32> [[TMP18]]) |
| ; P9CHECK-NEXT: [[TMP19:%.*]] = zext <8 x i16> [[WIDE_LOAD17]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP20:%.*]] = zext <8 x i16> [[WIDE_LOAD9]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw <8 x i32> [[TMP19]], [[TMP20]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI2]], <8 x i32> [[TMP21]]) |
| ; P9CHECK-NEXT: [[TMP22:%.*]] = zext <8 x i16> [[WIDE_LOAD18]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP23:%.*]] = zext <8 x i16> [[WIDE_LOAD10]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <8 x i32> [[TMP22]], [[TMP23]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI3]], <8 x i32> [[TMP24]]) |
| ; P9CHECK-NEXT: [[TMP25:%.*]] = zext <8 x i16> [[WIDE_LOAD19]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP26:%.*]] = zext <8 x i16> [[WIDE_LOAD11]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP27:%.*]] = mul nuw nsw <8 x i32> [[TMP25]], [[TMP26]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI4]], <8 x i32> [[TMP27]]) |
| ; P9CHECK-NEXT: [[TMP28:%.*]] = zext <8 x i16> [[WIDE_LOAD20]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP29:%.*]] = zext <8 x i16> [[WIDE_LOAD12]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw <8 x i32> [[TMP28]], [[TMP29]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI5]], <8 x i32> [[TMP30]]) |
| ; P9CHECK-NEXT: [[TMP31:%.*]] = zext <8 x i16> [[WIDE_LOAD21]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP32:%.*]] = zext <8 x i16> [[WIDE_LOAD13]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP33:%.*]] = mul nuw nsw <8 x i32> [[TMP31]], [[TMP32]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI6]], <8 x i32> [[TMP33]]) |
| ; P9CHECK-NEXT: [[TMP34:%.*]] = zext <8 x i16> [[WIDE_LOAD22]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP35:%.*]] = zext <8 x i16> [[WIDE_LOAD14]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP36:%.*]] = mul nuw nsw <8 x i32> [[TMP34]], [[TMP35]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI7]], <8 x i32> [[TMP36]]) |
| ; P9CHECK-NEXT: [[TMP37:%.*]] = zext <8 x i16> [[WIDE_LOAD23]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP38:%.*]] = zext <8 x i16> [[WIDE_LOAD15]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP39:%.*]] = mul nuw nsw <8 x i32> [[TMP37]], [[TMP38]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI8]], <8 x i32> [[TMP39]]) |
| ; P9CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64 |
| ; P9CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P9CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] |
| ; P9CHECK: [[MIDDLE_BLOCK]]: |
| ; P9CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]] |
| ; P9CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]] |
| ; P9CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]] |
| ; P9CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]] |
| ; P9CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]] |
| ; P9CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]] |
| ; P9CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]] |
| ; P9CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]]) |
| ; P9CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P9CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P9CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8 |
| ; P9CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF11]] |
| ; P9CHECK: [[VEC_EPILOG_PH]]: |
| ; P9CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7 |
| ; P9CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]] |
| ; P9CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P9CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P9CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <8 x i16>, ptr [[TMP44]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[TMP45:%.*]] = zext <8 x i16> [[WIDE_LOAD42]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP46:%.*]] = zext <8 x i16> [[WIDE_LOAD41]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP47:%.*]] = mul nuw nsw <8 x i32> [[TMP45]], [[TMP46]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI40]], <8 x i32> [[TMP47]]) |
| ; P9CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 8 |
| ; P9CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]] |
| ; P9CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] |
| ; P9CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P9CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]]) |
| ; P9CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]] |
| ; P9CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P9CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P9CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P9CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P9CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P9CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P9CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P9CHECK: [[FOR_COND_CLEANUP]]: |
| ; P9CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P9CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P9CHECK: [[FOR_BODY]]: |
| ; P9CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P9CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]] |
| ; P9CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[CONV:%.*]] = zext i16 [[TMP50]] to i32 |
| ; P9CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[I_09]] |
| ; P9CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[CONV2:%.*]] = zext i16 [[TMP51]] to i32 |
| ; P9CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]] |
| ; P9CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[MUL]], [[SUM_08]] |
| ; P9CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P9CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P9CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] |
| ; |
| ; P10CHECK-LABEL: define signext range(i32 0, -2147483648) i32 @dotu16( |
| ; P10CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] { |
| ; P10CHECK-NEXT: [[ENTRY:.*]]: |
| ; P10CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P10CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P10CHECK: [[ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 |
| ; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P10CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64 |
| ; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P10CHECK: [[VECTOR_PH]]: |
| ; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63 |
| ; P10CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; P10CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P10CHECK: [[VECTOR_BODY]]: |
| ; P10CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX]] |
| ; P10CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 8 |
| ; P10CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 16 |
| ; P10CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 24 |
| ; P10CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 32 |
| ; P10CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 40 |
| ; P10CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 48 |
| ; P10CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 56 |
| ; P10CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2, !tbaa [[SHORT_TBAA17:![0-9]+]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i16>, ptr [[TMP6]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[TMP7]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX]] |
| ; P10CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 8 |
| ; P10CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 16 |
| ; P10CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 24 |
| ; P10CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 32 |
| ; P10CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 40 |
| ; P10CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 48 |
| ; P10CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 56 |
| ; P10CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i16>, ptr [[TMP8]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[TMP9]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i16>, ptr [[TMP12]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i16>, ptr [[TMP13]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i16>, ptr [[TMP14]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i16>, ptr [[TMP15]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[TMP16:%.*]] = zext <8 x i16> [[WIDE_LOAD16]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP17:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <8 x i32> [[TMP16]], [[TMP17]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI]], <8 x i32> [[TMP18]]) |
| ; P10CHECK-NEXT: [[TMP19:%.*]] = zext <8 x i16> [[WIDE_LOAD17]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP20:%.*]] = zext <8 x i16> [[WIDE_LOAD9]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw <8 x i32> [[TMP19]], [[TMP20]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI2]], <8 x i32> [[TMP21]]) |
| ; P10CHECK-NEXT: [[TMP22:%.*]] = zext <8 x i16> [[WIDE_LOAD18]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP23:%.*]] = zext <8 x i16> [[WIDE_LOAD10]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <8 x i32> [[TMP22]], [[TMP23]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI3]], <8 x i32> [[TMP24]]) |
| ; P10CHECK-NEXT: [[TMP25:%.*]] = zext <8 x i16> [[WIDE_LOAD19]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP26:%.*]] = zext <8 x i16> [[WIDE_LOAD11]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP27:%.*]] = mul nuw nsw <8 x i32> [[TMP25]], [[TMP26]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI4]], <8 x i32> [[TMP27]]) |
| ; P10CHECK-NEXT: [[TMP28:%.*]] = zext <8 x i16> [[WIDE_LOAD20]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP29:%.*]] = zext <8 x i16> [[WIDE_LOAD12]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP30:%.*]] = mul nuw nsw <8 x i32> [[TMP28]], [[TMP29]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI5]], <8 x i32> [[TMP30]]) |
| ; P10CHECK-NEXT: [[TMP31:%.*]] = zext <8 x i16> [[WIDE_LOAD21]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP32:%.*]] = zext <8 x i16> [[WIDE_LOAD13]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP33:%.*]] = mul nuw nsw <8 x i32> [[TMP31]], [[TMP32]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI6]], <8 x i32> [[TMP33]]) |
| ; P10CHECK-NEXT: [[TMP34:%.*]] = zext <8 x i16> [[WIDE_LOAD22]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP35:%.*]] = zext <8 x i16> [[WIDE_LOAD14]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP36:%.*]] = mul nuw nsw <8 x i32> [[TMP34]], [[TMP35]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI7]], <8 x i32> [[TMP36]]) |
| ; P10CHECK-NEXT: [[TMP37:%.*]] = zext <8 x i16> [[WIDE_LOAD23]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP38:%.*]] = zext <8 x i16> [[WIDE_LOAD15]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP39:%.*]] = mul nuw nsw <8 x i32> [[TMP37]], [[TMP38]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI8]], <8 x i32> [[TMP39]]) |
| ; P10CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64 |
| ; P10CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P10CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] |
| ; P10CHECK: [[MIDDLE_BLOCK]]: |
| ; P10CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]] |
| ; P10CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]] |
| ; P10CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]] |
| ; P10CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]] |
| ; P10CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]] |
| ; P10CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]] |
| ; P10CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]] |
| ; P10CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]]) |
| ; P10CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P10CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P10CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8 |
| ; P10CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF20:![0-9]+]] |
| ; P10CHECK: [[VEC_EPILOG_PH]]: |
| ; P10CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7 |
| ; P10CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]] |
| ; P10CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P10CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P10CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <8 x i16>, ptr [[TMP44]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[TMP45:%.*]] = zext <8 x i16> [[WIDE_LOAD42]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP46:%.*]] = zext <8 x i16> [[WIDE_LOAD41]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP47:%.*]] = mul nuw nsw <8 x i32> [[TMP45]], [[TMP46]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI40]], <8 x i32> [[TMP47]]) |
| ; P10CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 8 |
| ; P10CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]] |
| ; P10CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] |
| ; P10CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P10CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]]) |
| ; P10CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]] |
| ; P10CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P10CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P10CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P10CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P10CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P10CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P10CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P10CHECK: [[FOR_COND_CLEANUP]]: |
| ; P10CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P10CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P10CHECK: [[FOR_BODY]]: |
| ; P10CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P10CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]] |
| ; P10CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[CONV:%.*]] = zext i16 [[TMP50]] to i32 |
| ; P10CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[I_09]] |
| ; P10CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[CONV2:%.*]] = zext i16 [[TMP51]] to i32 |
| ; P10CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV2]], [[CONV]] |
| ; P10CHECK-NEXT: [[ADD]] = add nuw nsw i32 [[MUL]], [[SUM_08]] |
| ; P10CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P10CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P10CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] |
| ; |
| entry: |
| %cmp7 = icmp sgt i64 %n, 0 |
| br i1 %cmp7, label %for.body, label %for.cond.cleanup |
| |
| for.cond.cleanup: ; preds = %for.body, %entry |
| %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ] |
| ret i32 %sum.0.lcssa |
| |
| for.body: ; preds = %entry, %for.body |
| %i.09 = phi i64 [ %inc, %for.body ], [ 0, %entry ] |
| %sum.08 = phi i32 [ %add, %for.body ], [ 0, %entry ] |
| %arrayidx = getelementptr inbounds nuw [2 x i8], ptr %a, i64 %i.09 |
| %0 = load i16, ptr %arrayidx, align 2, !tbaa !12 |
| %conv = zext i16 %0 to i32 |
| %arrayidx1 = getelementptr inbounds nuw [2 x i8], ptr %b, i64 %i.09 |
| %1 = load i16, ptr %arrayidx1, align 2, !tbaa !12 |
| %conv2 = zext i16 %1 to i32 |
| %mul = mul nuw nsw i32 %conv2, %conv |
| %add = add nuw nsw i32 %mul, %sum.08 |
| %inc = add nuw nsw i64 %i.09, 1 |
| %exitcond.not = icmp eq i64 %inc, %n |
| br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !14 |
| } |
| |
| ; Function Attrs: nofree norecurse nosync nounwind memory(argmem: read) uwtable |
| define signext i32 @dots16(ptr noundef readonly captures(none) %a, ptr noundef readonly captures(none) %b, i64 noundef %n) { |
| ; P8CHECK-LABEL: define signext i32 @dots16( |
| ; P8CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] { |
| ; P8CHECK-NEXT: [[ENTRY:.*]]: |
| ; P8CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P8CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P8CHECK: [[ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 |
| ; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P8CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64 |
| ; P8CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P8CHECK: [[VECTOR_PH]]: |
| ; P8CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63 |
| ; P8CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; P8CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P8CHECK: [[VECTOR_BODY]]: |
| ; P8CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX]] |
| ; P8CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 8 |
| ; P8CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 16 |
| ; P8CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 24 |
| ; P8CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 32 |
| ; P8CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 40 |
| ; P8CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 48 |
| ; P8CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 56 |
| ; P8CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i16>, ptr [[TMP6]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[TMP7]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX]] |
| ; P8CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 8 |
| ; P8CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 16 |
| ; P8CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 24 |
| ; P8CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 32 |
| ; P8CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 40 |
| ; P8CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 48 |
| ; P8CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 56 |
| ; P8CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i16>, ptr [[TMP8]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[TMP9]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i16>, ptr [[TMP12]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i16>, ptr [[TMP13]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i16>, ptr [[TMP14]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i16>, ptr [[TMP15]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[TMP16:%.*]] = sext <8 x i16> [[WIDE_LOAD16]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP17:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP18:%.*]] = mul nsw <8 x i32> [[TMP16]], [[TMP17]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI]], <8 x i32> [[TMP18]]) |
| ; P8CHECK-NEXT: [[TMP19:%.*]] = sext <8 x i16> [[WIDE_LOAD17]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP20:%.*]] = sext <8 x i16> [[WIDE_LOAD9]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP21:%.*]] = mul nsw <8 x i32> [[TMP19]], [[TMP20]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI2]], <8 x i32> [[TMP21]]) |
| ; P8CHECK-NEXT: [[TMP22:%.*]] = sext <8 x i16> [[WIDE_LOAD18]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP23:%.*]] = sext <8 x i16> [[WIDE_LOAD10]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP24:%.*]] = mul nsw <8 x i32> [[TMP22]], [[TMP23]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI3]], <8 x i32> [[TMP24]]) |
| ; P8CHECK-NEXT: [[TMP25:%.*]] = sext <8 x i16> [[WIDE_LOAD19]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP26:%.*]] = sext <8 x i16> [[WIDE_LOAD11]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP27:%.*]] = mul nsw <8 x i32> [[TMP25]], [[TMP26]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI4]], <8 x i32> [[TMP27]]) |
| ; P8CHECK-NEXT: [[TMP28:%.*]] = sext <8 x i16> [[WIDE_LOAD20]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP29:%.*]] = sext <8 x i16> [[WIDE_LOAD12]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP30:%.*]] = mul nsw <8 x i32> [[TMP28]], [[TMP29]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI5]], <8 x i32> [[TMP30]]) |
| ; P8CHECK-NEXT: [[TMP31:%.*]] = sext <8 x i16> [[WIDE_LOAD21]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP32:%.*]] = sext <8 x i16> [[WIDE_LOAD13]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP33:%.*]] = mul nsw <8 x i32> [[TMP31]], [[TMP32]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI6]], <8 x i32> [[TMP33]]) |
| ; P8CHECK-NEXT: [[TMP34:%.*]] = sext <8 x i16> [[WIDE_LOAD22]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP35:%.*]] = sext <8 x i16> [[WIDE_LOAD14]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP36:%.*]] = mul nsw <8 x i32> [[TMP34]], [[TMP35]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI7]], <8 x i32> [[TMP36]]) |
| ; P8CHECK-NEXT: [[TMP37:%.*]] = sext <8 x i16> [[WIDE_LOAD23]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP38:%.*]] = sext <8 x i16> [[WIDE_LOAD15]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP39:%.*]] = mul nsw <8 x i32> [[TMP37]], [[TMP38]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI8]], <8 x i32> [[TMP39]]) |
| ; P8CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64 |
| ; P8CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P8CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] |
| ; P8CHECK: [[MIDDLE_BLOCK]]: |
| ; P8CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]] |
| ; P8CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]] |
| ; P8CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]] |
| ; P8CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]] |
| ; P8CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]] |
| ; P8CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]] |
| ; P8CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]] |
| ; P8CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]]) |
| ; P8CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P8CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P8CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P8CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8 |
| ; P8CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF20]] |
| ; P8CHECK: [[VEC_EPILOG_PH]]: |
| ; P8CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7 |
| ; P8CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]] |
| ; P8CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P8CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P8CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P8CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P8CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]] |
| ; P8CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <8 x i16>, ptr [[TMP44]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[TMP45:%.*]] = sext <8 x i16> [[WIDE_LOAD42]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP46:%.*]] = sext <8 x i16> [[WIDE_LOAD41]] to <8 x i32> |
| ; P8CHECK-NEXT: [[TMP47:%.*]] = mul nsw <8 x i32> [[TMP45]], [[TMP46]] |
| ; P8CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI40]], <8 x i32> [[TMP47]]) |
| ; P8CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 8 |
| ; P8CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]] |
| ; P8CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] |
| ; P8CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P8CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]]) |
| ; P8CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]] |
| ; P8CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P8CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P8CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P8CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P8CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P8CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P8CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P8CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P8CHECK: [[FOR_COND_CLEANUP]]: |
| ; P8CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P8CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P8CHECK: [[FOR_BODY]]: |
| ; P8CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P8CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P8CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]] |
| ; P8CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP50]] to i32 |
| ; P8CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[I_09]] |
| ; P8CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P8CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP51]] to i32 |
| ; P8CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]] |
| ; P8CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]] |
| ; P8CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P8CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P8CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] |
| ; |
| ; P9CHECK-LABEL: define signext i32 @dots16( |
| ; P9CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] { |
| ; P9CHECK-NEXT: [[ENTRY:.*]]: |
| ; P9CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P9CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P9CHECK: [[ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 |
| ; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P9CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64 |
| ; P9CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P9CHECK: [[VECTOR_PH]]: |
| ; P9CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63 |
| ; P9CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; P9CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P9CHECK: [[VECTOR_BODY]]: |
| ; P9CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX]] |
| ; P9CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 8 |
| ; P9CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 16 |
| ; P9CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 24 |
| ; P9CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 32 |
| ; P9CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 40 |
| ; P9CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 48 |
| ; P9CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 56 |
| ; P9CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i16>, ptr [[TMP6]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[TMP7]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX]] |
| ; P9CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 8 |
| ; P9CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 16 |
| ; P9CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 24 |
| ; P9CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 32 |
| ; P9CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 40 |
| ; P9CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 48 |
| ; P9CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 56 |
| ; P9CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i16>, ptr [[TMP8]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[TMP9]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i16>, ptr [[TMP12]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i16>, ptr [[TMP13]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i16>, ptr [[TMP14]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i16>, ptr [[TMP15]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[TMP16:%.*]] = sext <8 x i16> [[WIDE_LOAD16]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP17:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP18:%.*]] = mul nsw <8 x i32> [[TMP16]], [[TMP17]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI]], <8 x i32> [[TMP18]]) |
| ; P9CHECK-NEXT: [[TMP19:%.*]] = sext <8 x i16> [[WIDE_LOAD17]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP20:%.*]] = sext <8 x i16> [[WIDE_LOAD9]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP21:%.*]] = mul nsw <8 x i32> [[TMP19]], [[TMP20]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI2]], <8 x i32> [[TMP21]]) |
| ; P9CHECK-NEXT: [[TMP22:%.*]] = sext <8 x i16> [[WIDE_LOAD18]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP23:%.*]] = sext <8 x i16> [[WIDE_LOAD10]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP24:%.*]] = mul nsw <8 x i32> [[TMP22]], [[TMP23]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI3]], <8 x i32> [[TMP24]]) |
| ; P9CHECK-NEXT: [[TMP25:%.*]] = sext <8 x i16> [[WIDE_LOAD19]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP26:%.*]] = sext <8 x i16> [[WIDE_LOAD11]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP27:%.*]] = mul nsw <8 x i32> [[TMP25]], [[TMP26]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI4]], <8 x i32> [[TMP27]]) |
| ; P9CHECK-NEXT: [[TMP28:%.*]] = sext <8 x i16> [[WIDE_LOAD20]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP29:%.*]] = sext <8 x i16> [[WIDE_LOAD12]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP30:%.*]] = mul nsw <8 x i32> [[TMP28]], [[TMP29]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI5]], <8 x i32> [[TMP30]]) |
| ; P9CHECK-NEXT: [[TMP31:%.*]] = sext <8 x i16> [[WIDE_LOAD21]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP32:%.*]] = sext <8 x i16> [[WIDE_LOAD13]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP33:%.*]] = mul nsw <8 x i32> [[TMP31]], [[TMP32]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI6]], <8 x i32> [[TMP33]]) |
| ; P9CHECK-NEXT: [[TMP34:%.*]] = sext <8 x i16> [[WIDE_LOAD22]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP35:%.*]] = sext <8 x i16> [[WIDE_LOAD14]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP36:%.*]] = mul nsw <8 x i32> [[TMP34]], [[TMP35]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI7]], <8 x i32> [[TMP36]]) |
| ; P9CHECK-NEXT: [[TMP37:%.*]] = sext <8 x i16> [[WIDE_LOAD23]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP38:%.*]] = sext <8 x i16> [[WIDE_LOAD15]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP39:%.*]] = mul nsw <8 x i32> [[TMP37]], [[TMP38]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI8]], <8 x i32> [[TMP39]]) |
| ; P9CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64 |
| ; P9CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P9CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] |
| ; P9CHECK: [[MIDDLE_BLOCK]]: |
| ; P9CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]] |
| ; P9CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]] |
| ; P9CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]] |
| ; P9CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]] |
| ; P9CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]] |
| ; P9CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]] |
| ; P9CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]] |
| ; P9CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]]) |
| ; P9CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P9CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P9CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P9CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8 |
| ; P9CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF11]] |
| ; P9CHECK: [[VEC_EPILOG_PH]]: |
| ; P9CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7 |
| ; P9CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]] |
| ; P9CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P9CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P9CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P9CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P9CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]] |
| ; P9CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <8 x i16>, ptr [[TMP44]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[TMP45:%.*]] = sext <8 x i16> [[WIDE_LOAD42]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP46:%.*]] = sext <8 x i16> [[WIDE_LOAD41]] to <8 x i32> |
| ; P9CHECK-NEXT: [[TMP47:%.*]] = mul nsw <8 x i32> [[TMP45]], [[TMP46]] |
| ; P9CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI40]], <8 x i32> [[TMP47]]) |
| ; P9CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 8 |
| ; P9CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]] |
| ; P9CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] |
| ; P9CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P9CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]]) |
| ; P9CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]] |
| ; P9CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P9CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P9CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P9CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P9CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P9CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P9CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P9CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P9CHECK: [[FOR_COND_CLEANUP]]: |
| ; P9CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P9CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P9CHECK: [[FOR_BODY]]: |
| ; P9CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P9CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P9CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]] |
| ; P9CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP50]] to i32 |
| ; P9CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[I_09]] |
| ; P9CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P9CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP51]] to i32 |
| ; P9CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]] |
| ; P9CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]] |
| ; P9CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P9CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P9CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] |
| ; |
| ; P10CHECK-LABEL: define signext i32 @dots16( |
| ; P10CHECK-SAME: ptr noundef readonly captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], i64 noundef [[N:%.*]]) #[[ATTR0]] { |
| ; P10CHECK-NEXT: [[ENTRY:.*]]: |
| ; P10CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 |
| ; P10CHECK-NEXT: br i1 [[CMP7]], label %[[ITER_CHECK:.*]], label %[[FOR_COND_CLEANUP:.*]] |
| ; P10CHECK: [[ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 |
| ; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; P10CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 64 |
| ; P10CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; P10CHECK: [[VECTOR_PH]]: |
| ; P10CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 63 |
| ; P10CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; P10CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; P10CHECK: [[VECTOR_BODY]]: |
| ; P10CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE24:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE25:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE26:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE27:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI6:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE28:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI7:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE29:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI8:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE30:%.*]], %[[VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX]] |
| ; P10CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 8 |
| ; P10CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 16 |
| ; P10CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 24 |
| ; P10CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 32 |
| ; P10CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 40 |
| ; P10CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 48 |
| ; P10CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP0]], i64 56 |
| ; P10CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <8 x i16>, ptr [[TMP4]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x i16>, ptr [[TMP5]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x i16>, ptr [[TMP6]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[TMP7]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX]] |
| ; P10CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 8 |
| ; P10CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 16 |
| ; P10CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 24 |
| ; P10CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 32 |
| ; P10CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 40 |
| ; P10CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 48 |
| ; P10CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i16, ptr [[TMP8]], i64 56 |
| ; P10CHECK-NEXT: [[WIDE_LOAD16:%.*]] = load <8 x i16>, ptr [[TMP8]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[TMP9]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x i16>, ptr [[TMP10]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD19:%.*]] = load <8 x i16>, ptr [[TMP11]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD20:%.*]] = load <8 x i16>, ptr [[TMP12]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i16>, ptr [[TMP13]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD22:%.*]] = load <8 x i16>, ptr [[TMP14]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <8 x i16>, ptr [[TMP15]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[TMP16:%.*]] = sext <8 x i16> [[WIDE_LOAD16]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP17:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP18:%.*]] = mul nsw <8 x i32> [[TMP16]], [[TMP17]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI]], <8 x i32> [[TMP18]]) |
| ; P10CHECK-NEXT: [[TMP19:%.*]] = sext <8 x i16> [[WIDE_LOAD17]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP20:%.*]] = sext <8 x i16> [[WIDE_LOAD9]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP21:%.*]] = mul nsw <8 x i32> [[TMP19]], [[TMP20]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE24]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI2]], <8 x i32> [[TMP21]]) |
| ; P10CHECK-NEXT: [[TMP22:%.*]] = sext <8 x i16> [[WIDE_LOAD18]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP23:%.*]] = sext <8 x i16> [[WIDE_LOAD10]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP24:%.*]] = mul nsw <8 x i32> [[TMP22]], [[TMP23]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE25]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI3]], <8 x i32> [[TMP24]]) |
| ; P10CHECK-NEXT: [[TMP25:%.*]] = sext <8 x i16> [[WIDE_LOAD19]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP26:%.*]] = sext <8 x i16> [[WIDE_LOAD11]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP27:%.*]] = mul nsw <8 x i32> [[TMP25]], [[TMP26]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE26]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI4]], <8 x i32> [[TMP27]]) |
| ; P10CHECK-NEXT: [[TMP28:%.*]] = sext <8 x i16> [[WIDE_LOAD20]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP29:%.*]] = sext <8 x i16> [[WIDE_LOAD12]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP30:%.*]] = mul nsw <8 x i32> [[TMP28]], [[TMP29]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE27]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI5]], <8 x i32> [[TMP30]]) |
| ; P10CHECK-NEXT: [[TMP31:%.*]] = sext <8 x i16> [[WIDE_LOAD21]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP32:%.*]] = sext <8 x i16> [[WIDE_LOAD13]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP33:%.*]] = mul nsw <8 x i32> [[TMP31]], [[TMP32]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE28]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI6]], <8 x i32> [[TMP33]]) |
| ; P10CHECK-NEXT: [[TMP34:%.*]] = sext <8 x i16> [[WIDE_LOAD22]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP35:%.*]] = sext <8 x i16> [[WIDE_LOAD14]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP36:%.*]] = mul nsw <8 x i32> [[TMP34]], [[TMP35]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE29]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI7]], <8 x i32> [[TMP36]]) |
| ; P10CHECK-NEXT: [[TMP37:%.*]] = sext <8 x i16> [[WIDE_LOAD23]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP38:%.*]] = sext <8 x i16> [[WIDE_LOAD15]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP39:%.*]] = mul nsw <8 x i32> [[TMP37]], [[TMP38]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE30]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI8]], <8 x i32> [[TMP39]]) |
| ; P10CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64 |
| ; P10CHECK-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; P10CHECK-NEXT: br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] |
| ; P10CHECK: [[MIDDLE_BLOCK]]: |
| ; P10CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE24]], [[PARTIAL_REDUCE]] |
| ; P10CHECK-NEXT: [[BIN_RDX31:%.*]] = add <4 x i32> [[PARTIAL_REDUCE25]], [[BIN_RDX]] |
| ; P10CHECK-NEXT: [[BIN_RDX32:%.*]] = add <4 x i32> [[PARTIAL_REDUCE26]], [[BIN_RDX31]] |
| ; P10CHECK-NEXT: [[BIN_RDX33:%.*]] = add <4 x i32> [[PARTIAL_REDUCE27]], [[BIN_RDX32]] |
| ; P10CHECK-NEXT: [[BIN_RDX34:%.*]] = add <4 x i32> [[PARTIAL_REDUCE28]], [[BIN_RDX33]] |
| ; P10CHECK-NEXT: [[BIN_RDX35:%.*]] = add <4 x i32> [[PARTIAL_REDUCE29]], [[BIN_RDX34]] |
| ; P10CHECK-NEXT: [[BIN_RDX36:%.*]] = add <4 x i32> [[PARTIAL_REDUCE30]], [[BIN_RDX35]] |
| ; P10CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX36]]) |
| ; P10CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; P10CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; P10CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; P10CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8 |
| ; P10CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF20]] |
| ; P10CHECK: [[VEC_EPILOG_PH]]: |
| ; P10CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[N_MOD_VF37:%.*]] = and i64 [[N]], 7 |
| ; P10CHECK-NEXT: [[N_VEC38:%.*]] = sub i64 [[N]], [[N_MOD_VF37]] |
| ; P10CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; P10CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; P10CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; P10CHECK-NEXT: [[INDEX39:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT44:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[VEC_PHI40:%.*]] = phi <4 x i32> [ [[TMP52]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE43:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; P10CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[INDEX39]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD41:%.*]] = load <8 x i16>, ptr [[TMP43]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[INDEX39]] |
| ; P10CHECK-NEXT: [[WIDE_LOAD42:%.*]] = load <8 x i16>, ptr [[TMP44]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[TMP45:%.*]] = sext <8 x i16> [[WIDE_LOAD42]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP46:%.*]] = sext <8 x i16> [[WIDE_LOAD41]] to <8 x i32> |
| ; P10CHECK-NEXT: [[TMP47:%.*]] = mul nsw <8 x i32> [[TMP45]], [[TMP46]] |
| ; P10CHECK-NEXT: [[PARTIAL_REDUCE43]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v8i32(<4 x i32> [[VEC_PHI40]], <8 x i32> [[TMP47]]) |
| ; P10CHECK-NEXT: [[INDEX_NEXT44]] = add nuw i64 [[INDEX39]], 8 |
| ; P10CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT44]], [[N_VEC38]] |
| ; P10CHECK-NEXT: br i1 [[TMP48]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] |
| ; P10CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; P10CHECK-NEXT: [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE43]]) |
| ; P10CHECK-NEXT: [[CMP_N45:%.*]] = icmp eq i64 [[N]], [[N_VEC38]] |
| ; P10CHECK-NEXT: br i1 [[CMP_N45]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; P10CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; P10CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC38]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P10CHECK-NEXT: [[BC_MERGE_RDX45:%.*]] = phi i32 [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP41]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] |
| ; P10CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; P10CHECK: [[FOR_COND_CLEANUP_LOOPEXIT]]: |
| ; P10CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD:%.*]], %[[FOR_BODY]] ], [ [[TMP41]], %[[MIDDLE_BLOCK]] ], [ [[TMP49]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ] |
| ; P10CHECK-NEXT: br label %[[FOR_COND_CLEANUP]] |
| ; P10CHECK: [[FOR_COND_CLEANUP]]: |
| ; P10CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_COND_CLEANUP_LOOPEXIT]] ] |
| ; P10CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] |
| ; P10CHECK: [[FOR_BODY]]: |
| ; P10CHECK-NEXT: [[I_09:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P10CHECK-NEXT: [[SUM_08:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX45]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; P10CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[A]], i64 [[I_09]] |
| ; P10CHECK-NEXT: [[TMP50:%.*]] = load i16, ptr [[ARRAYIDX]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP50]] to i32 |
| ; P10CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw [2 x i8], ptr [[B]], i64 [[I_09]] |
| ; P10CHECK-NEXT: [[TMP51:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2, !tbaa [[SHORT_TBAA17]] |
| ; P10CHECK-NEXT: [[CONV2:%.*]] = sext i16 [[TMP51]] to i32 |
| ; P10CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]] |
| ; P10CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_08]] |
| ; P10CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_09]], 1 |
| ; P10CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] |
| ; P10CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] |
| ; |
| entry: |
| %cmp7 = icmp sgt i64 %n, 0 |
| br i1 %cmp7, label %for.body, label %for.cond.cleanup |
| |
| for.cond.cleanup: ; preds = %for.body, %entry |
| %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ] |
| ret i32 %sum.0.lcssa |
| |
| for.body: ; preds = %entry, %for.body |
| %i.09 = phi i64 [ %inc, %for.body ], [ 0, %entry ] |
| %sum.08 = phi i32 [ %add, %for.body ], [ 0, %entry ] |
| %arrayidx = getelementptr inbounds nuw [2 x i8], ptr %a, i64 %i.09 |
| %0 = load i16, ptr %arrayidx, align 2, !tbaa !12 |
| %conv = sext i16 %0 to i32 |
| %arrayidx1 = getelementptr inbounds nuw [2 x i8], ptr %b, i64 %i.09 |
| %1 = load i16, ptr %arrayidx1, align 2, !tbaa !12 |
| %conv2 = sext i16 %1 to i32 |
| %mul = mul nsw i32 %conv2, %conv |
| %add = add nsw i32 %mul, %sum.08 |
| %inc = add nuw nsw i64 %i.09, 1 |
| %exitcond.not = icmp eq i64 %inc, %n |
| br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !15 |
| } |
| |
| !6 = !{!"omnipotent char", !7, i64 0} |
| !7 = !{!"Simple C/C++ TBAA"} |
| !8 = !{!6, !6, i64 0} |
| !9 = distinct !{!9, !10} |
| !10 = !{!"llvm.loop.mustprogress"} |
| !11 = distinct !{!11, !10} |
| !12 = !{!13, !13, i64 0} |
| !13 = !{!"short", !6, i64 0} |
| !14 = distinct !{!14, !10} |
| !15 = distinct !{!15, !10} |
| ;. |
| ; P8CHECK: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0} |
| ; P8CHECK: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0} |
| ; P8CHECK: [[META2]] = !{!"Simple C/C++ TBAA"} |
| ; P8CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]], [[META6:![0-9]+]]} |
| ; P8CHECK: [[META4]] = !{!"llvm.loop.mustprogress"} |
| ; P8CHECK: [[META5]] = !{!"llvm.loop.isvectorized", i32 1} |
| ; P8CHECK: [[META6]] = !{!"llvm.loop.unroll.runtime.disable"} |
| ; P8CHECK: [[PROF7]] = !{!"branch_weights", i32 16, i32 112} |
| ; P8CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]], [[META5]], [[META6]]} |
| ; P8CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]], [[META6]], [[META5]]} |
| ; P8CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]], [[META5]], [[META6]]} |
| ; P8CHECK: [[PROF11]] = !{!"branch_weights", i32 4, i32 60} |
| ; P8CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META4]], [[META5]], [[META6]]} |
| ; P8CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META4]], [[META6]], [[META5]]} |
| ; P8CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META5]], [[META6]]} |
| ; P8CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]], [[META6]]} |
| ; P8CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META6]], [[META5]]} |
| ; P8CHECK: [[SHORT_TBAA17]] = !{[[META18:![0-9]+]], [[META18]], i64 0} |
| ; P8CHECK: [[META18]] = !{!"short", [[META1]], i64 0} |
| ; P8CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META4]], [[META5]], [[META6]]} |
| ; P8CHECK: [[PROF20]] = !{!"branch_weights", i32 8, i32 56} |
| ; P8CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META4]], [[META5]], [[META6]]} |
| ; P8CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META4]], [[META6]], [[META5]]} |
| ; P8CHECK: [[LOOP23]] = distinct !{[[LOOP23]], [[META4]], [[META5]], [[META6]]} |
| ; P8CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META4]], [[META5]], [[META6]]} |
| ; P8CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META4]], [[META6]], [[META5]]} |
| ;. |
| ; P9CHECK: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0} |
| ; P9CHECK: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0} |
| ; P9CHECK: [[META2]] = !{!"Simple C/C++ TBAA"} |
| ; P9CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]], [[META6:![0-9]+]]} |
| ; P9CHECK: [[META4]] = !{!"llvm.loop.mustprogress"} |
| ; P9CHECK: [[META5]] = !{!"llvm.loop.isvectorized", i32 1} |
| ; P9CHECK: [[META6]] = !{!"llvm.loop.unroll.runtime.disable"} |
| ; P9CHECK: [[PROF7]] = !{!"branch_weights", i32 16, i32 112} |
| ; P9CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]], [[META5]], [[META6]]} |
| ; P9CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]], [[META6]], [[META5]]} |
| ; P9CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]], [[META5]], [[META6]]} |
| ; P9CHECK: [[PROF11]] = !{!"branch_weights", i32 8, i32 56} |
| ; P9CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META4]], [[META5]], [[META6]]} |
| ; P9CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META4]], [[META6]], [[META5]]} |
| ; P9CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META5]], [[META6]]} |
| ; P9CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]], [[META6]]} |
| ; P9CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META6]], [[META5]]} |
| ; P9CHECK: [[SHORT_TBAA17]] = !{[[META18:![0-9]+]], [[META18]], i64 0} |
| ; P9CHECK: [[META18]] = !{!"short", [[META1]], i64 0} |
| ; P9CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META4]], [[META5]], [[META6]]} |
| ; P9CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META4]], [[META5]], [[META6]]} |
| ; P9CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META4]], [[META6]], [[META5]]} |
| ; P9CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META4]], [[META5]], [[META6]]} |
| ; P9CHECK: [[LOOP23]] = distinct !{[[LOOP23]], [[META4]], [[META5]], [[META6]]} |
| ; P9CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META4]], [[META6]], [[META5]]} |
| ;. |
| ; P10CHECK: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0} |
| ; P10CHECK: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0} |
| ; P10CHECK: [[META2]] = !{!"Simple C/C++ TBAA"} |
| ; P10CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]], [[META6:![0-9]+]]} |
| ; P10CHECK: [[META4]] = !{!"llvm.loop.mustprogress"} |
| ; P10CHECK: [[META5]] = !{!"llvm.loop.isvectorized", i32 1} |
| ; P10CHECK: [[META6]] = !{!"llvm.loop.unroll.runtime.disable"} |
| ; P10CHECK: [[PROF7]] = !{!"branch_weights", i32 16, i32 112} |
| ; P10CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]], [[META5]], [[META6]]} |
| ; P10CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]], [[META6]], [[META5]]} |
| ; P10CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]], [[META5]], [[META6]]} |
| ; P10CHECK: [[PROF11]] = !{!"branch_weights", i32 4, i32 60} |
| ; P10CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META4]], [[META5]], [[META6]]} |
| ; P10CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META4]], [[META6]], [[META5]]} |
| ; P10CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META5]], [[META6]]} |
| ; P10CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]], [[META6]]} |
| ; P10CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META6]], [[META5]]} |
| ; P10CHECK: [[SHORT_TBAA17]] = !{[[META18:![0-9]+]], [[META18]], i64 0} |
| ; P10CHECK: [[META18]] = !{!"short", [[META1]], i64 0} |
| ; P10CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META4]], [[META5]], [[META6]]} |
| ; P10CHECK: [[PROF20]] = !{!"branch_weights", i32 8, i32 56} |
| ; P10CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META4]], [[META5]], [[META6]]} |
| ; P10CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META4]], [[META6]], [[META5]]} |
| ; P10CHECK: [[LOOP23]] = distinct !{[[LOOP23]], [[META4]], [[META5]], [[META6]]} |
| ; P10CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META4]], [[META5]], [[META6]]} |
| ; P10CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META4]], [[META6]], [[META5]]} |
| ;. |
