| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 6 |
| ; RUN: opt -p loop-vectorize -force-target-instruction-cost=1 -S %s | FileCheck --check-prefixes=COMMON,COST1 %s |
| ; RUN: opt -p loop-vectorize -force-target-instruction-cost=10 -S %s | FileCheck --check-prefixes=COMMON,COST10 %s |
| |
| target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128-Fn32" |
| target triple = "arm64-apple-macosx14.0.0" |
| |
| define double @test_reduction_costs() { |
| ; COMMON-LABEL: define double @test_reduction_costs() { |
| ; COMMON-NEXT: [[ENTRY:.*:]] |
| ; COMMON-NEXT: br label %[[VECTOR_PH:.*]] |
| ; COMMON: [[VECTOR_PH]]: |
| ; COMMON-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; COMMON: [[VECTOR_BODY]]: |
| ; COMMON-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP0:%.*]], %[[VECTOR_BODY]] ] |
| ; COMMON-NEXT: [[VEC_PHI1:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ] |
| ; COMMON-NEXT: [[TMP0]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI]], <2 x double> splat (double 3.000000e+00)) |
| ; COMMON-NEXT: [[TMP1]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI1]], <2 x double> splat (double 9.000000e+00)) |
| ; COMMON-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; COMMON: [[MIDDLE_BLOCK]]: |
| ; COMMON-NEXT: br label %[[EXIT:.*]] |
| ; COMMON: [[EXIT]]: |
| ; COMMON-NEXT: [[DIV:%.*]] = fmul double [[TMP0]], [[TMP1]] |
| ; COMMON-NEXT: ret double [[DIV]] |
| ; |
| entry: |
| br label %loop.1 |
| |
| loop.1: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.1 ] |
| %r.1 = phi double [ 0.000000e+00, %entry ], [ %r.1.next, %loop.1 ] |
| %r.2 = phi double [ 0.000000e+00, %entry ], [ %r.2.next, %loop.1 ] |
| %r.1.next = fadd double %r.1, 3.000000e+00 |
| %r.2.next = fadd double %r.2, 9.000000e+00 |
| %iv.next = add i64 %iv, 1 |
| %ec = icmp eq i64 %iv, 1 |
| br i1 %ec, label %exit, label %loop.1 |
| |
| exit: |
| %div = fmul double %r.1.next, %r.2.next |
| ret double %div |
| } |
| |
| define void @test_iv_cost(ptr %ptr.start, i8 %a, i64 %b) { |
| ; COST1-LABEL: define void @test_iv_cost( |
| ; COST1-SAME: ptr [[PTR_START:%.*]], i8 [[A:%.*]], i64 [[B:%.*]]) { |
| ; COST1-NEXT: [[ENTRY:.*:]] |
| ; COST1-NEXT: [[A_EXT:%.*]] = zext i8 [[A]] to i64 |
| ; COST1-NEXT: [[START:%.*]] = call i64 @llvm.umin.i64(i64 [[B]], i64 [[A_EXT]]) |
| ; COST1-NEXT: [[C:%.*]] = icmp eq i64 [[START]], 0 |
| ; COST1-NEXT: br i1 [[C]], [[EXIT:label %.*]], label %[[ITER_CHECK:.*]] |
| ; COST1: [[ITER_CHECK]]: |
| ; COST1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[START]], 4 |
| ; COST1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; COST1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; COST1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[START]], 32 |
| ; COST1-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; COST1: [[VECTOR_PH]]: |
| ; COST1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[START]], 32 |
| ; COST1-NEXT: [[N_VEC:%.*]] = sub i64 [[START]], [[N_MOD_VF]] |
| ; COST1-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; COST1: [[VECTOR_BODY]]: |
| ; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; COST1-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[INDEX]] |
| ; COST1-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 16 |
| ; COST1-NEXT: store <16 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 1 |
| ; COST1-NEXT: store <16 x i8> zeroinitializer, ptr [[TMP0]], align 1 |
| ; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32 |
| ; COST1-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; COST1-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] |
| ; COST1: [[MIDDLE_BLOCK]]: |
| ; COST1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[START]], [[N_VEC]] |
| ; COST1-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; COST1: [[VEC_EPILOG_ITER_CHECK]]: |
| ; COST1-NEXT: [[IND_END:%.*]] = sub i64 [[START]], [[N_VEC]] |
| ; COST1-NEXT: [[IND_END9:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC]] |
| ; COST1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4 |
| ; COST1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF4:![0-9]+]] |
| ; COST1: [[VEC_EPILOG_PH]]: |
| ; COST1-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; COST1-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[START]], 4 |
| ; COST1-NEXT: [[N_VEC3:%.*]] = sub i64 [[START]], [[N_MOD_VF2]] |
| ; COST1-NEXT: [[TMP2:%.*]] = sub i64 [[START]], [[N_VEC3]] |
| ; COST1-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC3]] |
| ; COST1-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; COST1: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; COST1-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; COST1-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[INDEX4]] |
| ; COST1-NEXT: store <4 x i8> zeroinitializer, ptr [[NEXT_GEP5]], align 1 |
| ; COST1-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 4 |
| ; COST1-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]] |
| ; COST1-NEXT: br i1 [[TMP4]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] |
| ; COST1: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; COST1-NEXT: [[CMP_N7:%.*]] = icmp eq i64 [[START]], [[N_VEC3]] |
| ; COST1-NEXT: br i1 [[CMP_N7]], [[EXIT_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; COST1: [[VEC_EPILOG_SCALAR_PH]]: |
| ; |
| ; COST10-LABEL: define void @test_iv_cost( |
| ; COST10-SAME: ptr [[PTR_START:%.*]], i8 [[A:%.*]], i64 [[B:%.*]]) { |
| ; COST10-NEXT: [[ENTRY:.*:]] |
| ; COST10-NEXT: [[A_EXT:%.*]] = zext i8 [[A]] to i64 |
| ; COST10-NEXT: [[START:%.*]] = call i64 @llvm.umin.i64(i64 [[B]], i64 [[A_EXT]]) |
| ; COST10-NEXT: [[C:%.*]] = icmp eq i64 [[START]], 0 |
| ; COST10-NEXT: br i1 [[C]], [[EXIT:label %.*]], label %[[ITER_CHECK:.*]] |
| ; COST10: [[ITER_CHECK]]: |
| ; COST10-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[START]], 4 |
| ; COST10-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; COST10: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; COST10-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[START]], 16 |
| ; COST10-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; COST10: [[VECTOR_PH]]: |
| ; COST10-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[START]], 16 |
| ; COST10-NEXT: [[N_VEC:%.*]] = sub i64 [[START]], [[N_MOD_VF]] |
| ; COST10-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; COST10: [[VECTOR_BODY]]: |
| ; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; COST10-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[INDEX]] |
| ; COST10-NEXT: store <16 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 1 |
| ; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 |
| ; COST10-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; COST10-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] |
| ; COST10: [[MIDDLE_BLOCK]]: |
| ; COST10-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[START]], [[N_VEC]] |
| ; COST10-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; COST10: [[VEC_EPILOG_ITER_CHECK]]: |
| ; COST10-NEXT: [[IND_END:%.*]] = sub i64 [[START]], [[N_VEC]] |
| ; COST10-NEXT: [[IND_END9:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC]] |
| ; COST10-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4 |
| ; COST10-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF4:![0-9]+]] |
| ; COST10: [[VEC_EPILOG_PH]]: |
| ; COST10-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; COST10-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[START]], 4 |
| ; COST10-NEXT: [[N_VEC3:%.*]] = sub i64 [[START]], [[N_MOD_VF2]] |
| ; COST10-NEXT: [[TMP1:%.*]] = sub i64 [[START]], [[N_VEC3]] |
| ; COST10-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC3]] |
| ; COST10-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; COST10: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; COST10-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; COST10-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[INDEX4]] |
| ; COST10-NEXT: store <4 x i8> zeroinitializer, ptr [[NEXT_GEP5]], align 1 |
| ; COST10-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 4 |
| ; COST10-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]] |
| ; COST10-NEXT: br i1 [[TMP3]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] |
| ; COST10: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; COST10-NEXT: [[CMP_N7:%.*]] = icmp eq i64 [[START]], [[N_VEC3]] |
| ; COST10-NEXT: br i1 [[CMP_N7]], [[EXIT_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; COST10: [[VEC_EPILOG_SCALAR_PH]]: |
| ; |
| entry: |
| %a.ext = zext i8 %a to i64 |
| %start = call i64 @llvm.umin.i64(i64 %b, i64 %a.ext) |
| %c = icmp eq i64 %start, 0 |
| br i1 %c, label %exit, label %loop |
| |
| loop: |
| %iv = phi i64 [ %start, %entry ], [ %iv.next, %loop ] |
| %ptr.iv = phi ptr [ %ptr.start, %entry ], [ %ptr.iv.next, %loop ] |
| %iv.next = add i64 %iv, -1 |
| %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 1 |
| store i8 0, ptr %ptr.iv, align 1 |
| %ec = icmp eq i64 %iv.next, 0 |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| define void @test_exit_branch_cost(ptr %dst, ptr noalias %x.ptr, ptr noalias %y.ptr, ptr noalias %dst.1, i1 %c.4, ptr %src, ptr noalias %dst.3, i1 %c.3, ptr noalias %dst.2) { |
| ; COMMON-LABEL: define void @test_exit_branch_cost( |
| ; COMMON-SAME: ptr [[DST:%.*]], ptr noalias [[X_PTR:%.*]], ptr noalias [[Y_PTR:%.*]], ptr noalias [[DST_1:%.*]], i1 [[C_4:%.*]], ptr [[SRC:%.*]], ptr noalias [[DST_3:%.*]], i1 [[C_3:%.*]], ptr noalias [[DST_2:%.*]]) { |
| ; COMMON-NEXT: [[ENTRY:.*:]] |
| ; COMMON-NEXT: br label %[[VECTOR_MEMCHECK:.*]] |
| ; COMMON: [[VECTOR_MEMCHECK]]: |
| ; COMMON-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8 |
| ; COMMON-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 8 |
| ; COMMON-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]] |
| ; COMMON-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]] |
| ; COMMON-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] |
| ; COMMON-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; COMMON: [[VECTOR_PH]]: |
| ; COMMON-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C_3]], i64 0 |
| ; COMMON-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer |
| ; COMMON-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <2 x i1> poison, i1 [[C_4]], i64 0 |
| ; COMMON-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT2]], <2 x i1> poison, <2 x i32> zeroinitializer |
| ; COMMON-NEXT: [[TMP0:%.*]] = select i1 [[C_4]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i1> zeroinitializer |
| ; COMMON-NEXT: [[TMP1:%.*]] = xor <2 x i1> [[TMP0]], splat (i1 true) |
| ; COMMON-NEXT: [[TMP2:%.*]] = select <2 x i1> [[BROADCAST_SPLAT]], <2 x i1> [[BROADCAST_SPLAT3]], <2 x i1> zeroinitializer |
| ; COMMON-NEXT: [[TMP3:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT3]], splat (i1 true) |
| ; COMMON-NEXT: [[TMP4:%.*]] = select <2 x i1> [[BROADCAST_SPLAT]], <2 x i1> [[TMP3]], <2 x i1> zeroinitializer |
| ; COMMON-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; COMMON: [[VECTOR_BODY]]: |
| ; COMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE17:.*]] ] |
| ; COMMON-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[X_PTR]], i64 [[INDEX]] |
| ; COMMON-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8 |
| ; COMMON-NEXT: [[TMP6:%.*]] = icmp eq <2 x i64> [[WIDE_LOAD]], zeroinitializer |
| ; COMMON-NEXT: [[TMP7:%.*]] = xor <2 x i1> [[TMP6]], splat (i1 true) |
| ; COMMON-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0 |
| ; COMMON-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] |
| ; COMMON: [[PRED_STORE_IF]]: |
| ; COMMON-NEXT: store i64 0, ptr [[DST_1]], align 8 |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE]] |
| ; COMMON: [[PRED_STORE_CONTINUE]]: |
| ; COMMON-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1 |
| ; COMMON-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]] |
| ; COMMON: [[PRED_STORE_IF4]]: |
| ; COMMON-NEXT: store i64 0, ptr [[DST_1]], align 8 |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE5]] |
| ; COMMON: [[PRED_STORE_CONTINUE5]]: |
| ; COMMON-NEXT: [[TMP10:%.*]] = select <2 x i1> [[TMP7]], <2 x i1> [[TMP1]], <2 x i1> zeroinitializer |
| ; COMMON-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP10]], i32 0 |
| ; COMMON-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]] |
| ; COMMON: [[PRED_STORE_IF6]]: |
| ; COMMON-NEXT: store i64 0, ptr [[DST_3]], align 8 |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE7]] |
| ; COMMON: [[PRED_STORE_CONTINUE7]]: |
| ; COMMON-NEXT: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP10]], i32 1 |
| ; COMMON-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]] |
| ; COMMON: [[PRED_STORE_IF8]]: |
| ; COMMON-NEXT: store i64 0, ptr [[DST_3]], align 8 |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE9]] |
| ; COMMON: [[PRED_STORE_CONTINUE9]]: |
| ; COMMON-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP7]], <2 x i1> [[TMP2]], <2 x i1> zeroinitializer |
| ; COMMON-NEXT: [[TMP14:%.*]] = or <2 x i1> [[TMP6]], [[TMP13]] |
| ; COMMON-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP13]], <2 x i64> zeroinitializer, <2 x i64> splat (i64 1) |
| ; COMMON-NEXT: [[TMP15:%.*]] = extractelement <2 x i1> [[TMP14]], i32 0 |
| ; COMMON-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]] |
| ; COMMON: [[PRED_STORE_IF10]]: |
| ; COMMON-NEXT: [[TMP16:%.*]] = extractelement <2 x i64> [[PREDPHI]], i32 0 |
| ; COMMON-NEXT: store i64 [[TMP16]], ptr [[DST_2]], align 8 |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE11]] |
| ; COMMON: [[PRED_STORE_CONTINUE11]]: |
| ; COMMON-NEXT: [[TMP17:%.*]] = extractelement <2 x i1> [[TMP14]], i32 1 |
| ; COMMON-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]] |
| ; COMMON: [[PRED_STORE_IF12]]: |
| ; COMMON-NEXT: [[TMP18:%.*]] = extractelement <2 x i64> [[PREDPHI]], i32 1 |
| ; COMMON-NEXT: store i64 [[TMP18]], ptr [[DST_2]], align 8 |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE13]] |
| ; COMMON: [[PRED_STORE_CONTINUE13]]: |
| ; COMMON-NEXT: [[TMP19:%.*]] = select <2 x i1> [[TMP7]], <2 x i1> [[TMP4]], <2 x i1> zeroinitializer |
| ; COMMON-NEXT: [[TMP20:%.*]] = or <2 x i1> [[TMP14]], [[TMP19]] |
| ; COMMON-NEXT: [[TMP21:%.*]] = extractelement <2 x i1> [[TMP20]], i32 0 |
| ; COMMON-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15:.*]] |
| ; COMMON: [[PRED_STORE_IF14]]: |
| ; COMMON-NEXT: [[TMP22:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META7:![0-9]+]] |
| ; COMMON-NEXT: store i64 [[TMP22]], ptr [[DST]], align 8, !alias.scope [[META10:![0-9]+]], !noalias [[META7]] |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE15]] |
| ; COMMON: [[PRED_STORE_CONTINUE15]]: |
| ; COMMON-NEXT: [[TMP23:%.*]] = extractelement <2 x i1> [[TMP20]], i32 1 |
| ; COMMON-NEXT: br i1 [[TMP23]], label %[[PRED_STORE_IF16:.*]], label %[[PRED_STORE_CONTINUE17]] |
| ; COMMON: [[PRED_STORE_IF16]]: |
| ; COMMON-NEXT: [[TMP24:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META7]] |
| ; COMMON-NEXT: store i64 [[TMP24]], ptr [[DST]], align 8, !alias.scope [[META10]], !noalias [[META7]] |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE17]] |
| ; COMMON: [[PRED_STORE_CONTINUE17]]: |
| ; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; COMMON-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64 |
| ; COMMON-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] |
| ; COMMON: [[MIDDLE_BLOCK]]: |
| ; COMMON-NEXT: br label %[[SCALAR_PH]] |
| ; COMMON: [[SCALAR_PH]]: |
| ; |
| entry: |
| br label %loop.header |
| |
| loop.header: |
| %iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ] |
| %x.gep = getelementptr i64, ptr %x.ptr, i64 %iv |
| %x = load i64, ptr %x.gep |
| %y.gep = getelementptr i32, ptr %y.ptr, i64 %iv |
| %y = load i32, ptr %y.gep |
| %c1 = icmp eq i64 %x, 0 |
| br i1 %c1, label %then.4, label %then.1 |
| |
| then.1: |
| %and32831 = and i32 %y, 1 |
| store i64 0, ptr %dst.1, align 8 |
| %c.2 = icmp eq i32 %y, 0 |
| %or.cond = select i1 %c.4, i1 %c.3, i1 false |
| br i1 %or.cond, label %then.2, label %else.1 |
| |
| else.1: ; preds = %then.1 |
| store i64 0, ptr %dst.3, align 8 |
| br label %then.2 |
| |
| then.2: |
| br i1 %c.3, label %then.3, label %loop.latch |
| |
| then.3: |
| br i1 %c.4, label %then.5, label %else.2 |
| |
| then.4: |
| call void @llvm.assume(i1 %c.4) |
| br label %then.5 |
| |
| then.5: |
| %1 = phi i64 [ 1, %then.4 ], [ 0, %then.3 ] |
| store i64 %1, ptr %dst.2, align 8 |
| br label %else.2 |
| |
| else.2: |
| %l = load i64, ptr %src, align 8 |
| store i64 %l, ptr %dst, align 8 |
| br label %loop.latch |
| |
| loop.latch: |
| %iv.next = add i64 %iv, 1 |
| %ec = icmp eq i64 %iv, 64 |
| br i1 %ec, label %exit, label %loop.header |
| |
| exit: |
| ret void |
| } |
| |
| define void @invalid_legacy_cost(i64 %N, ptr %x) #0 { |
| ; COST1-LABEL: define void @invalid_legacy_cost( |
| ; COST1-SAME: i64 [[N:%.*]], ptr [[X:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; COST1-NEXT: [[ENTRY:.*:]] |
| ; COST1-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 |
| ; COST1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4 |
| ; COST1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; COST1: [[VECTOR_PH]]: |
| ; COST1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4 |
| ; COST1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]] |
| ; COST1-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; COST1: [[VECTOR_BODY]]: |
| ; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; COST1-NEXT: [[TMP1:%.*]] = alloca i8, i64 0, align 16 |
| ; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP1]], i64 0 |
| ; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer |
| ; COST1-NEXT: [[TMP2:%.*]] = getelementptr ptr, ptr [[X]], i64 [[INDEX]] |
| ; COST1-NEXT: [[TMP3:%.*]] = getelementptr ptr, ptr [[TMP2]], i64 2 |
| ; COST1-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 8 |
| ; COST1-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 8 |
| ; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 |
| ; COST1-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; COST1-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] |
| ; COST1: [[MIDDLE_BLOCK]]: |
| ; COST1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] |
| ; COST1-NEXT: br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]] |
| ; COST1: [[SCALAR_PH]]: |
| ; |
| ; COST10-LABEL: define void @invalid_legacy_cost( |
| ; COST10-SAME: i64 [[N:%.*]], ptr [[X:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; COST10-NEXT: [[ENTRY:.*:]] |
| ; COST10-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 |
| ; COST10-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2 |
| ; COST10-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; COST10: [[VECTOR_PH]]: |
| ; COST10-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2 |
| ; COST10-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]] |
| ; COST10-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; COST10: [[VECTOR_BODY]]: |
| ; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; COST10-NEXT: [[TMP1:%.*]] = alloca i8, i64 0, align 16 |
| ; COST10-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP1]], i64 0 |
| ; COST10-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer |
| ; COST10-NEXT: [[TMP2:%.*]] = getelementptr ptr, ptr [[X]], i64 [[INDEX]] |
| ; COST10-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 8 |
| ; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; COST10-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; COST10-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] |
| ; COST10: [[MIDDLE_BLOCK]]: |
| ; COST10-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] |
| ; COST10-NEXT: br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]] |
| ; COST10: [[SCALAR_PH]]: |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] |
| %0 = alloca i8, i64 0, align 16 |
| %arrayidx = getelementptr ptr, ptr %x, i64 %iv |
| store ptr %0, ptr %arrayidx, align 8 |
| %iv.next = add i64 %iv, 1 |
| %exitcond.not = icmp eq i64 %iv, %N |
| br i1 %exitcond.not, label %for.end, label %for.body |
| |
| for.end: |
| ret void |
| } |
| |
| define void @loop_with_freeze_and_conditional_srem(ptr %dst, ptr %keyinfo, ptr %invariant.ptr, i32 %divisor) { |
| ; COMMON-LABEL: define void @loop_with_freeze_and_conditional_srem( |
| ; COMMON-SAME: ptr [[DST:%.*]], ptr [[KEYINFO:%.*]], ptr [[INVARIANT_PTR:%.*]], i32 [[DIVISOR:%.*]]) { |
| ; COMMON-NEXT: [[ENTRY:.*]]: |
| ; COMMON-NEXT: br label %[[LOOP:.*]] |
| ; COMMON: [[LOOP]]: |
| ; COMMON-NEXT: [[INDEX_NEXT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] |
| ; COMMON-NEXT: [[LOADED:%.*]] = load i32, ptr [[INVARIANT_PTR]], align 4 |
| ; COMMON-NEXT: [[FROZEN:%.*]] = freeze i32 [[LOADED]] |
| ; COMMON-NEXT: [[CMP:%.*]] = icmp eq i32 [[FROZEN]], 0 |
| ; COMMON-NEXT: br i1 [[CMP]], label %[[IF_ZERO:.*]], label %[[IF_NONZERO:.*]] |
| ; COMMON: [[IF_ZERO]]: |
| ; COMMON-NEXT: store i32 0, ptr [[KEYINFO]], align 4 |
| ; COMMON-NEXT: br label %[[LOOP_LATCH]] |
| ; COMMON: [[IF_NONZERO]]: |
| ; COMMON-NEXT: [[TMP11:%.*]] = srem i32 1, [[DIVISOR]] |
| ; COMMON-NEXT: store i32 [[TMP11]], ptr [[DST]], align 4 |
| ; COMMON-NEXT: br label %[[LOOP_LATCH]] |
| ; COMMON: [[LOOP_LATCH]]: |
| ; COMMON-NEXT: [[IV_NEXT]] = add i64 [[INDEX_NEXT]], 1 |
| ; COMMON-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32 |
| ; COMMON-NEXT: br i1 [[TMP16]], label %[[EXIT:.*]], label %[[LOOP]] |
| ; COMMON: [[EXIT]]: |
| ; COMMON-NEXT: ret void |
| ; |
| entry: |
| br label %loop |
| |
| loop: ; preds = %loop.latch, %entry |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] |
| %loaded = load i32, ptr %invariant.ptr, align 4 |
| %frozen = freeze i32 %loaded |
| %cmp = icmp eq i32 %frozen, 0 |
| br i1 %cmp, label %if.zero, label %if.nonzero |
| |
| if.zero: ; preds = %loop |
| store i32 0, ptr %keyinfo, align 4 |
| br label %loop.latch |
| |
| if.nonzero: ; preds = %loop |
| %rem = srem i32 1, %divisor |
| store i32 %rem, ptr %dst, align 4 |
| br label %loop.latch |
| |
| loop.latch: ; preds = %if.nonzero, %if.zero |
| %iv.next = add i64 %iv, 1 |
| %exitcond = icmp eq i64 %iv, 32 |
| br i1 %exitcond, label %exit, label %loop |
| |
| exit: ; preds = %loop.latch |
| ret void |
| } |
| |
| define void @interleave_group(ptr %dst) #1 { |
| ; COST1-LABEL: define void @interleave_group( |
| ; COST1-SAME: ptr [[DST:%.*]]) #[[ATTR1:[0-9]+]] { |
| ; COST1-NEXT: [[ITER_CHECK:.*:]] |
| ; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; COST1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; COST1: [[VECTOR_PH]]: |
| ; COST1-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; COST1: [[VECTOR_BODY]]: |
| ; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; COST1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 16 |
| ; COST1-NEXT: [[TMP1:%.*]] = mul i64 [[INDEX]], 3 |
| ; COST1-NEXT: [[TMP2:%.*]] = mul i64 [[TMP0]], 3 |
| ; COST1-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]] |
| ; COST1-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]] |
| ; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP3]], align 1 |
| ; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP4]], align 1 |
| ; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32 |
| ; COST1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96 |
| ; COST1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] |
| ; COST1: [[MIDDLE_BLOCK]]: |
| ; COST1-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; COST1: [[VEC_EPILOG_ITER_CHECK]]: |
| ; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF4]] |
| ; COST1: [[VEC_EPILOG_PH]]: |
| ; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0 |
| ; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer |
| ; COST1-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3> |
| ; COST1-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; COST1: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; COST1-NEXT: [[INDEX1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; COST1-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; COST1-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], splat (i64 3) |
| ; COST1-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i32 0 |
| ; COST1-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP6]], i32 1 |
| ; COST1-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP6]], i32 2 |
| ; COST1-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP6]], i32 3 |
| ; COST1-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]] |
| ; COST1-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]] |
| ; COST1-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP9]] |
| ; COST1-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP10]] |
| ; COST1-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[TMP11]], i64 2 |
| ; COST1-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP12]], i64 2 |
| ; COST1-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[TMP13]], i64 2 |
| ; COST1-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[TMP14]], i64 2 |
| ; COST1-NEXT: store i8 0, ptr [[TMP15]], align 1 |
| ; COST1-NEXT: store i8 0, ptr [[TMP16]], align 1 |
| ; COST1-NEXT: store i8 0, ptr [[TMP17]], align 1 |
| ; COST1-NEXT: store i8 0, ptr [[TMP18]], align 1 |
| ; COST1-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[TMP11]], i64 1 |
| ; COST1-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[TMP12]], i64 1 |
| ; COST1-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP13]], i64 1 |
| ; COST1-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[TMP14]], i64 1 |
| ; COST1-NEXT: store i8 0, ptr [[TMP19]], align 1 |
| ; COST1-NEXT: store i8 0, ptr [[TMP20]], align 1 |
| ; COST1-NEXT: store i8 0, ptr [[TMP21]], align 1 |
| ; COST1-NEXT: store i8 0, ptr [[TMP22]], align 1 |
| ; COST1-NEXT: store i8 0, ptr [[TMP11]], align 1 |
| ; COST1-NEXT: store i8 0, ptr [[TMP12]], align 1 |
| ; COST1-NEXT: store i8 0, ptr [[TMP13]], align 1 |
| ; COST1-NEXT: store i8 0, ptr [[TMP14]], align 1 |
| ; COST1-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4 |
| ; COST1-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4) |
| ; COST1-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 100 |
| ; COST1-NEXT: br i1 [[TMP23]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] |
| ; COST1: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; COST1-NEXT: br i1 false, [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; COST1: [[VEC_EPILOG_SCALAR_PH]]: |
| ; |
| ; COST10-LABEL: define void @interleave_group( |
| ; COST10-SAME: ptr [[DST:%.*]]) #[[ATTR1:[0-9]+]] { |
| ; COST10-NEXT: [[ITER_CHECK:.*:]] |
| ; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; COST10: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; COST10: [[VECTOR_PH]]: |
| ; COST10-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; COST10: [[VECTOR_BODY]]: |
| ; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; COST10-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 3 |
| ; COST10-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]] |
| ; COST10-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP1]], align 1 |
| ; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 |
| ; COST10-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96 |
| ; COST10-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] |
| ; COST10: [[MIDDLE_BLOCK]]: |
| ; COST10-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; COST10: [[VEC_EPILOG_ITER_CHECK]]: |
| ; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF4]] |
| ; COST10: [[VEC_EPILOG_PH]]: |
| ; COST10-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; COST10-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0 |
| ; COST10-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer |
| ; COST10-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3> |
| ; COST10-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; COST10: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; COST10-NEXT: [[INDEX1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; COST10-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; COST10-NEXT: [[TMP3:%.*]] = mul <4 x i64> [[VEC_IND]], splat (i64 3) |
| ; COST10-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0 |
| ; COST10-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1 |
| ; COST10-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2 |
| ; COST10-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3 |
| ; COST10-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP4]] |
| ; COST10-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP5]] |
| ; COST10-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]] |
| ; COST10-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]] |
| ; COST10-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP8]], i64 2 |
| ; COST10-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP9]], i64 2 |
| ; COST10-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP10]], i64 2 |
| ; COST10-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[TMP11]], i64 2 |
| ; COST10-NEXT: store i8 0, ptr [[TMP12]], align 1 |
| ; COST10-NEXT: store i8 0, ptr [[TMP13]], align 1 |
| ; COST10-NEXT: store i8 0, ptr [[TMP14]], align 1 |
| ; COST10-NEXT: store i8 0, ptr [[TMP15]], align 1 |
| ; COST10-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP8]], i64 1 |
| ; COST10-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[TMP9]], i64 1 |
| ; COST10-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[TMP10]], i64 1 |
| ; COST10-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[TMP11]], i64 1 |
| ; COST10-NEXT: store i8 0, ptr [[TMP16]], align 1 |
| ; COST10-NEXT: store i8 0, ptr [[TMP17]], align 1 |
| ; COST10-NEXT: store i8 0, ptr [[TMP18]], align 1 |
| ; COST10-NEXT: store i8 0, ptr [[TMP19]], align 1 |
| ; COST10-NEXT: store i8 0, ptr [[TMP8]], align 1 |
| ; COST10-NEXT: store i8 0, ptr [[TMP9]], align 1 |
| ; COST10-NEXT: store i8 0, ptr [[TMP10]], align 1 |
| ; COST10-NEXT: store i8 0, ptr [[TMP11]], align 1 |
| ; COST10-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4 |
| ; COST10-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4) |
| ; COST10-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 100 |
| ; COST10-NEXT: br i1 [[TMP20]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] |
| ; COST10: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; COST10-NEXT: br i1 false, [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; COST10: [[VEC_EPILOG_SCALAR_PH]]: |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %iv.3 = mul i64 %iv, 3 |
| %gep.0 = getelementptr i8, ptr %dst, i64 %iv.3 |
| %gep.2 = getelementptr i8, ptr %gep.0, i64 2 |
| store i8 0, ptr %gep.2, align 1 |
| %gep.1 = getelementptr i8, ptr %gep.0, i64 1 |
| store i8 0, ptr %gep.1, align 1 |
| store i8 0, ptr %gep.0, align 1 |
| %iv.next = add i64 %iv, 1 |
| %ec = icmp eq i64 %iv, 100 |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| define void @forced_scalar_instr(ptr %gep.dst) { |
| ; COMMON-LABEL: define void @forced_scalar_instr( |
| ; COMMON-SAME: ptr [[GEP_DST:%.*]]) { |
| ; COMMON-NEXT: [[ENTRY:.*:]] |
| ; COMMON-NEXT: br label %[[VECTOR_PH:.*]] |
| ; COMMON: [[VECTOR_PH]]: |
| ; COMMON-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; COMMON: [[VECTOR_BODY]]: |
| ; COMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ] |
| ; COMMON-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ] |
| ; COMMON-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i32 |
| ; COMMON-NEXT: [[TMP1:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 4) |
| ; COMMON-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0 |
| ; COMMON-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] |
| ; COMMON: [[PRED_STORE_IF]]: |
| ; COMMON-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 |
| ; COMMON-NEXT: [[TMP4:%.*]] = add i32 [[TMP0]], 0 |
| ; COMMON-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[GEP_DST]], i64 [[TMP3]] |
| ; COMMON-NEXT: [[TMP6:%.*]] = or i32 [[TMP4]], 1 |
| ; COMMON-NEXT: store i32 [[TMP6]], ptr [[TMP5]], align 4 |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE]] |
| ; COMMON: [[PRED_STORE_CONTINUE]]: |
| ; COMMON-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1 |
| ; COMMON-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]] |
| ; COMMON: [[PRED_STORE_IF1]]: |
| ; COMMON-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1 |
| ; COMMON-NEXT: [[TMP9:%.*]] = add i32 [[TMP0]], 1 |
| ; COMMON-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[GEP_DST]], i64 [[TMP8]] |
| ; COMMON-NEXT: [[TMP11:%.*]] = or i32 [[TMP9]], 1 |
| ; COMMON-NEXT: store i32 [[TMP11]], ptr [[TMP10]], align 4 |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE2]] |
| ; COMMON: [[PRED_STORE_CONTINUE2]]: |
| ; COMMON-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2 |
| ; COMMON-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]] |
| ; COMMON: [[PRED_STORE_IF3]]: |
| ; COMMON-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], 2 |
| ; COMMON-NEXT: [[TMP14:%.*]] = add i32 [[TMP0]], 2 |
| ; COMMON-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[GEP_DST]], i64 [[TMP13]] |
| ; COMMON-NEXT: [[TMP16:%.*]] = or i32 [[TMP14]], 1 |
| ; COMMON-NEXT: store i32 [[TMP16]], ptr [[TMP15]], align 4 |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE4]] |
| ; COMMON: [[PRED_STORE_CONTINUE4]]: |
| ; COMMON-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3 |
| ; COMMON-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]] |
| ; COMMON: [[PRED_STORE_IF5]]: |
| ; COMMON-NEXT: [[TMP18:%.*]] = add i64 [[INDEX]], 3 |
| ; COMMON-NEXT: [[TMP19:%.*]] = add i32 [[TMP0]], 3 |
| ; COMMON-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[GEP_DST]], i64 [[TMP18]] |
| ; COMMON-NEXT: [[TMP21:%.*]] = or i32 [[TMP19]], 1 |
| ; COMMON-NEXT: store i32 [[TMP21]], ptr [[TMP20]], align 4 |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE6]] |
| ; COMMON: [[PRED_STORE_CONTINUE6]]: |
| ; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 |
| ; COMMON-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4) |
| ; COMMON-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8 |
| ; COMMON-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] |
| ; COMMON: [[MIDDLE_BLOCK]]: |
| ; COMMON-NEXT: br label %[[EXIT:.*]] |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %gep = getelementptr i32, ptr %gep.dst, i64 %iv |
| %t = trunc i64 %iv to i32 |
| %o = or i32 %t, 1 |
| store i32 %o, ptr %gep, align 4 |
| %iv.next = add i64 %iv, 1 |
| %ec = icmp eq i64 %iv, 4 |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| attributes #0 = { "target-features"="+neon,+sve" vscale_range(1,16) } |
| attributes #1 = { "target-cpu"="neoverse-512tvb" } |
| |
| declare void @llvm.assume(i1 noundef) |
| declare i64 @llvm.umin.i64(i64, i64) |
