| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 |
| ; RUN: opt -p loop-vectorize -force-target-instruction-cost=1 -S %s | FileCheck %s |
| |
| target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128-Fn32" |
| target triple = "arm64-apple-macosx14.0.0" |
| |
| define double @test_reduction_costs() { |
| ; CHECK-LABEL: define double @test_reduction_costs() { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP0:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[TMP0]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI]], <2 x double> splat (double 3.000000e+00)) |
| ; CHECK-NEXT: [[TMP1]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI1]], <2 x double> splat (double 9.000000e+00)) |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; CHECK-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 2, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ] |
| ; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi double [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ] |
| ; CHECK-NEXT: br label %[[LOOP_1:.*]] |
| ; CHECK: [[LOOP_1]]: |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_1]] ] |
| ; CHECK-NEXT: [[R_1:%.*]] = phi double [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[R_1_NEXT:%.*]], %[[LOOP_1]] ] |
| ; CHECK-NEXT: [[R_2:%.*]] = phi double [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[R_2_NEXT:%.*]], %[[LOOP_1]] ] |
| ; CHECK-NEXT: [[R_1_NEXT]] = fadd double [[R_1]], 3.000000e+00 |
| ; CHECK-NEXT: [[R_2_NEXT]] = fadd double [[R_2]], 9.000000e+00 |
| ; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 1 |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_1]], !llvm.loop [[LOOP3:![0-9]+]] |
| ; CHECK: [[EXIT]]: |
| ; CHECK-NEXT: [[R_1_NEXT_LCSSA:%.*]] = phi double [ [[R_1_NEXT]], %[[LOOP_1]] ], [ [[TMP0]], %[[MIDDLE_BLOCK]] ] |
| ; CHECK-NEXT: [[R_2_NEXT_LCSSA:%.*]] = phi double [ [[R_2_NEXT]], %[[LOOP_1]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ] |
| ; CHECK-NEXT: [[DIV:%.*]] = fmul double [[R_1_NEXT_LCSSA]], [[R_2_NEXT_LCSSA]] |
| ; CHECK-NEXT: ret double [[DIV]] |
| ; |
| entry: |
| br label %loop.1 |
| |
| loop.1: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.1 ] |
| %r.1 = phi double [ 0.000000e+00, %entry ], [ %r.1.next, %loop.1 ] |
| %r.2 = phi double [ 0.000000e+00, %entry ], [ %r.2.next, %loop.1 ] |
| %r.1.next = fadd double %r.1, 3.000000e+00 |
| %r.2.next = fadd double %r.2, 9.000000e+00 |
| %iv.next = add i64 %iv, 1 |
| %ec = icmp eq i64 %iv, 1 |
| br i1 %ec, label %exit, label %loop.1 |
| |
| exit: |
| %div = fmul double %r.1.next, %r.2.next |
| ret double %div |
| } |
| |
| define void @test_iv_cost(ptr %ptr.start, i8 %a, i64 %b) { |
| ; CHECK-LABEL: define void @test_iv_cost( |
| ; CHECK-SAME: ptr [[PTR_START:%.*]], i8 [[A:%.*]], i64 [[B:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[A_EXT:%.*]] = zext i8 [[A]] to i64 |
| ; CHECK-NEXT: [[START:%.*]] = call i64 @llvm.umin.i64(i64 [[B]], i64 [[A_EXT]]) |
| ; CHECK-NEXT: [[C:%.*]] = icmp eq i64 [[START]], 0 |
| ; CHECK-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[ITER_CHECK:.*]] |
| ; CHECK: [[ITER_CHECK]]: |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[START]], 4 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[START]], 32 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[START]], 32 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[START]], [[N_MOD_VF]] |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX1]], 0 |
| ; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[TMP5]] |
| ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i32 0 |
| ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i32 16 |
| ; CHECK-NEXT: store <16 x i8> zeroinitializer, ptr [[TMP6]], align 1 |
| ; CHECK-NEXT: store <16 x i8> zeroinitializer, ptr [[TMP3]], align 1 |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 32 |
| ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[START]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; CHECK-NEXT: [[IND_END:%.*]] = sub i64 [[START]], [[N_VEC]] |
| ; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC]] |
| ; CHECK-NEXT: [[N_VEC_REMAINING:%.*]] = sub i64 [[START]], [[N_VEC]] |
| ; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4 |
| ; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]] |
| ; CHECK: [[VEC_EPILOG_PH]]: |
| ; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[START]], 4 |
| ; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[START]], [[N_MOD_VF2]] |
| ; CHECK-NEXT: [[IND_END1:%.*]] = sub i64 [[START]], [[N_VEC3]] |
| ; CHECK-NEXT: [[IND_END5:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC3]] |
| ; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT10:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 |
| ; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[TMP0]] |
| ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0 |
| ; CHECK-NEXT: store <4 x i8> zeroinitializer, ptr [[TMP2]], align 1 |
| ; CHECK-NEXT: [[INDEX_NEXT10]] = add nuw i64 [[INDEX]], 4 |
| ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT10]], [[N_VEC3]] |
| ; CHECK-NEXT: br i1 [[TMP7]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] |
| ; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N11:%.*]] = icmp eq i64 [[START]], [[N_VEC3]] |
| ; CHECK-NEXT: br i1 [[CMP_N11]], label %[[EXIT_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END1]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ] |
| ; CHECK-NEXT: [[BC_RESUME_VAL9:%.*]] = phi ptr [ [[IND_END5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END2]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR_START]], %[[ITER_CHECK]] ] |
| ; CHECK-NEXT: br label %[[LOOP:.*]] |
| ; CHECK: [[LOOP]]: |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL9]], %[[VEC_EPILOG_SCALAR_PH]] ] |
| ; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1 |
| ; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 1 |
| ; CHECK-NEXT: store i8 0, ptr [[PTR_IV]], align 1 |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 0 |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]] |
| ; CHECK: [[EXIT_LOOPEXIT]]: |
| ; CHECK-NEXT: br label %[[EXIT]] |
| ; CHECK: [[EXIT]]: |
| ; CHECK-NEXT: ret void |
| ; |
| entry: |
| %a.ext = zext i8 %a to i64 |
| %start = call i64 @llvm.umin.i64(i64 %b, i64 %a.ext) |
| %c = icmp eq i64 %start, 0 |
| br i1 %c, label %exit, label %loop |
| |
| loop: |
| %iv = phi i64 [ %start, %entry ], [ %iv.next, %loop ] |
| %ptr.iv = phi ptr [ %ptr.start, %entry ], [ %ptr.iv.next, %loop ] |
| %iv.next = add i64 %iv, -1 |
| %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 1 |
| store i8 0, ptr %ptr.iv, align 1 |
| %ec = icmp eq i64 %iv.next, 0 |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| define void @test_exit_branch_cost(ptr %dst, ptr noalias %x.ptr, ptr noalias %y.ptr, ptr %dst.1, i1 %c.4, ptr %src, ptr %dst.3, i1 %c.3, ptr %dst.2) { |
| ; CHECK-LABEL: define void @test_exit_branch_cost( |
| ; CHECK-SAME: ptr [[DST:%.*]], ptr noalias [[X_PTR:%.*]], ptr noalias [[Y_PTR:%.*]], ptr [[DST_1:%.*]], i1 [[C_4:%.*]], ptr [[SRC:%.*]], ptr [[DST_3:%.*]], i1 [[C_3:%.*]], ptr [[DST_2:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] |
| ; CHECK: [[VECTOR_MEMCHECK]]: |
| ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST_1]], i64 8 |
| ; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST_3]], i64 8 |
| ; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[DST_2]], i64 8 |
| ; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[DST]], i64 8 |
| ; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 8 |
| ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST_1]], [[SCEVGEP1]] |
| ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[DST_3]], [[SCEVGEP]] |
| ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] |
| ; CHECK-NEXT: [[BOUND05:%.*]] = icmp ult ptr [[DST_1]], [[SCEVGEP2]] |
| ; CHECK-NEXT: [[BOUND16:%.*]] = icmp ult ptr [[DST_2]], [[SCEVGEP]] |
| ; CHECK-NEXT: [[FOUND_CONFLICT7:%.*]] = and i1 [[BOUND05]], [[BOUND16]] |
| ; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT7]] |
| ; CHECK-NEXT: [[BOUND08:%.*]] = icmp ult ptr [[DST_1]], [[SCEVGEP3]] |
| ; CHECK-NEXT: [[BOUND19:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP]] |
| ; CHECK-NEXT: [[FOUND_CONFLICT10:%.*]] = and i1 [[BOUND08]], [[BOUND19]] |
| ; CHECK-NEXT: [[CONFLICT_RDX21:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT10]] |
| ; CHECK-NEXT: [[BOUND012:%.*]] = icmp ult ptr [[DST_1]], [[SCEVGEP4]] |
| ; CHECK-NEXT: [[BOUND113:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]] |
| ; CHECK-NEXT: [[FOUND_CONFLICT14:%.*]] = and i1 [[BOUND012]], [[BOUND113]] |
| ; CHECK-NEXT: [[CONFLICT_RDX15:%.*]] = or i1 [[CONFLICT_RDX21]], [[FOUND_CONFLICT14]] |
| ; CHECK-NEXT: [[BOUND016:%.*]] = icmp ult ptr [[DST_3]], [[SCEVGEP2]] |
| ; CHECK-NEXT: [[BOUND117:%.*]] = icmp ult ptr [[DST_2]], [[SCEVGEP1]] |
| ; CHECK-NEXT: [[FOUND_CONFLICT18:%.*]] = and i1 [[BOUND016]], [[BOUND117]] |
| ; CHECK-NEXT: [[CONFLICT_RDX19:%.*]] = or i1 [[CONFLICT_RDX15]], [[FOUND_CONFLICT18]] |
| ; CHECK-NEXT: [[BOUND020:%.*]] = icmp ult ptr [[DST_3]], [[SCEVGEP3]] |
| ; CHECK-NEXT: [[BOUND121:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]] |
| ; CHECK-NEXT: [[FOUND_CONFLICT22:%.*]] = and i1 [[BOUND020]], [[BOUND121]] |
| ; CHECK-NEXT: [[CONFLICT_RDX41:%.*]] = or i1 [[CONFLICT_RDX19]], [[FOUND_CONFLICT22]] |
| ; CHECK-NEXT: [[BOUND024:%.*]] = icmp ult ptr [[DST_3]], [[SCEVGEP4]] |
| ; CHECK-NEXT: [[BOUND125:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP1]] |
| ; CHECK-NEXT: [[FOUND_CONFLICT26:%.*]] = and i1 [[BOUND024]], [[BOUND125]] |
| ; CHECK-NEXT: [[CONFLICT_RDX27:%.*]] = or i1 [[CONFLICT_RDX41]], [[FOUND_CONFLICT26]] |
| ; CHECK-NEXT: [[BOUND028:%.*]] = icmp ult ptr [[DST_2]], [[SCEVGEP3]] |
| ; CHECK-NEXT: [[BOUND129:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]] |
| ; CHECK-NEXT: [[FOUND_CONFLICT30:%.*]] = and i1 [[BOUND028]], [[BOUND129]] |
| ; CHECK-NEXT: [[CONFLICT_RDX65:%.*]] = or i1 [[CONFLICT_RDX27]], [[FOUND_CONFLICT30]] |
| ; CHECK-NEXT: [[BOUND032:%.*]] = icmp ult ptr [[DST_2]], [[SCEVGEP4]] |
| ; CHECK-NEXT: [[BOUND133:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP2]] |
| ; CHECK-NEXT: [[FOUND_CONFLICT68:%.*]] = and i1 [[BOUND032]], [[BOUND133]] |
| ; CHECK-NEXT: [[CONFLICT_RDX35:%.*]] = or i1 [[CONFLICT_RDX65]], [[FOUND_CONFLICT68]] |
| ; CHECK-NEXT: [[BOUND036:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP4]] |
| ; CHECK-NEXT: [[BOUND137:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP3]] |
| ; CHECK-NEXT: [[FOUND_CONFLICT38:%.*]] = and i1 [[BOUND036]], [[BOUND137]] |
| ; CHECK-NEXT: [[CONFLICT_RDX39:%.*]] = or i1 [[CONFLICT_RDX35]], [[FOUND_CONFLICT38]] |
| ; CHECK-NEXT: br i1 [[CONFLICT_RDX39]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT40:%.*]] = insertelement <2 x i1> poison, i1 [[C_3]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT41:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT40]], <2 x i1> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT56:%.*]] = insertelement <2 x i1> poison, i1 [[C_4]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT57:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT56]], <2 x i1> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C_4]], <2 x i1> [[BROADCAST_SPLAT41]], <2 x i1> zeroinitializer |
| ; CHECK-NEXT: [[TMP11:%.*]] = xor <2 x i1> [[TMP1]], splat (i1 true) |
| ; CHECK-NEXT: [[TMP33:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT57]], splat (i1 true) |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE55:.*]] ] |
| ; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 |
| ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[X_PTR]], i64 [[TMP3]] |
| ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP4]], i32 0 |
| ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP6]], align 8 |
| ; CHECK-NEXT: [[TMP47:%.*]] = icmp eq <2 x i64> [[WIDE_LOAD]], zeroinitializer |
| ; CHECK-NEXT: [[TMP5:%.*]] = xor <2 x i1> [[TMP47]], splat (i1 true) |
| ; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0 |
| ; CHECK-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] |
| ; CHECK: [[PRED_STORE_IF]]: |
| ; CHECK-NEXT: store i64 0, ptr [[DST_1]], align 8, !alias.scope [[META7:![0-9]+]], !noalias [[META10:![0-9]+]] |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]] |
| ; CHECK: [[PRED_STORE_CONTINUE]]: |
| ; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP5]], i32 1 |
| ; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF42:.*]], label %[[PRED_STORE_CONTINUE43:.*]] |
| ; CHECK: [[PRED_STORE_IF42]]: |
| ; CHECK-NEXT: store i64 0, ptr [[DST_1]], align 8, !alias.scope [[META7]], !noalias [[META10]] |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE43]] |
| ; CHECK: [[PRED_STORE_CONTINUE43]]: |
| ; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP5]], <2 x i1> [[TMP11]], <2 x i1> zeroinitializer |
| ; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i1> [[TMP13]], i32 0 |
| ; CHECK-NEXT: br i1 [[TMP16]], label %[[PRED_STORE_IF44:.*]], label %[[PRED_STORE_CONTINUE45:.*]] |
| ; CHECK: [[PRED_STORE_IF44]]: |
| ; CHECK-NEXT: store i64 0, ptr [[DST_3]], align 8, !alias.scope [[META15:![0-9]+]], !noalias [[META16:![0-9]+]] |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE45]] |
| ; CHECK: [[PRED_STORE_CONTINUE45]]: |
| ; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i1> [[TMP13]], i32 1 |
| ; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF46:.*]], label %[[PRED_STORE_CONTINUE47:.*]] |
| ; CHECK: [[PRED_STORE_IF46]]: |
| ; CHECK-NEXT: store i64 0, ptr [[DST_3]], align 8, !alias.scope [[META15]], !noalias [[META16]] |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE47]] |
| ; CHECK: [[PRED_STORE_CONTINUE47]]: |
| ; CHECK-NEXT: [[TMP19:%.*]] = select <2 x i1> [[TMP5]], <2 x i1> [[BROADCAST_SPLAT41]], <2 x i1> zeroinitializer |
| ; CHECK-NEXT: [[TMP21:%.*]] = select <2 x i1> [[TMP19]], <2 x i1> [[BROADCAST_SPLAT57]], <2 x i1> zeroinitializer |
| ; CHECK-NEXT: [[TMP23:%.*]] = or <2 x i1> [[TMP47]], [[TMP21]] |
| ; CHECK-NEXT: [[PREDPHI58:%.*]] = select <2 x i1> [[TMP21]], <2 x i64> zeroinitializer, <2 x i64> splat (i64 1) |
| ; CHECK-NEXT: [[TMP28:%.*]] = extractelement <2 x i1> [[TMP23]], i32 0 |
| ; CHECK-NEXT: br i1 [[TMP28]], label %[[PRED_STORE_IF48:.*]], label %[[PRED_STORE_CONTINUE49:.*]] |
| ; CHECK: [[PRED_STORE_IF48]]: |
| ; CHECK-NEXT: [[TMP29:%.*]] = extractelement <2 x i64> [[PREDPHI58]], i32 0 |
| ; CHECK-NEXT: store i64 [[TMP29]], ptr [[DST_2]], align 8, !alias.scope [[META17:![0-9]+]], !noalias [[META18:![0-9]+]] |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE49]] |
| ; CHECK: [[PRED_STORE_CONTINUE49]]: |
| ; CHECK-NEXT: [[TMP30:%.*]] = extractelement <2 x i1> [[TMP23]], i32 1 |
| ; CHECK-NEXT: br i1 [[TMP30]], label %[[PRED_STORE_IF50:.*]], label %[[PRED_STORE_CONTINUE51:.*]] |
| ; CHECK: [[PRED_STORE_IF50]]: |
| ; CHECK-NEXT: [[TMP31:%.*]] = extractelement <2 x i64> [[PREDPHI58]], i32 1 |
| ; CHECK-NEXT: store i64 [[TMP31]], ptr [[DST_2]], align 8, !alias.scope [[META17]], !noalias [[META18]] |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE51]] |
| ; CHECK: [[PRED_STORE_CONTINUE51]]: |
| ; CHECK-NEXT: [[TMP35:%.*]] = select <2 x i1> [[TMP19]], <2 x i1> [[TMP33]], <2 x i1> zeroinitializer |
| ; CHECK-NEXT: [[TMP37:%.*]] = or <2 x i1> [[TMP23]], [[TMP35]] |
| ; CHECK-NEXT: [[TMP42:%.*]] = extractelement <2 x i1> [[TMP37]], i32 0 |
| ; CHECK-NEXT: br i1 [[TMP42]], label %[[PRED_STORE_IF52:.*]], label %[[PRED_STORE_CONTINUE53:.*]] |
| ; CHECK: [[PRED_STORE_IF52]]: |
| ; CHECK-NEXT: [[TMP24:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META19:![0-9]+]] |
| ; CHECK-NEXT: store i64 [[TMP24]], ptr [[DST]], align 8, !alias.scope [[META20:![0-9]+]], !noalias [[META19]] |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE53]] |
| ; CHECK: [[PRED_STORE_CONTINUE53]]: |
| ; CHECK-NEXT: [[TMP44:%.*]] = extractelement <2 x i1> [[TMP37]], i32 1 |
| ; CHECK-NEXT: br i1 [[TMP44]], label %[[PRED_STORE_IF54:.*]], label %[[PRED_STORE_CONTINUE55]] |
| ; CHECK: [[PRED_STORE_IF54]]: |
| ; CHECK-NEXT: [[TMP25:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META19]] |
| ; CHECK-NEXT: store i64 [[TMP25]], ptr [[DST]], align 8, !alias.scope [[META20]], !noalias [[META19]] |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE55]] |
| ; CHECK: [[PRED_STORE_CONTINUE55]]: |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; CHECK-NEXT: [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64 |
| ; CHECK-NEXT: br i1 [[TMP46]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 64, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ] |
| ; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] |
| ; CHECK: [[LOOP_HEADER]]: |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] |
| ; CHECK-NEXT: [[X_GEP:%.*]] = getelementptr i64, ptr [[X_PTR]], i64 [[IV]] |
| ; CHECK-NEXT: [[X:%.*]] = load i64, ptr [[X_GEP]], align 8 |
| ; CHECK-NEXT: [[Y_GEP:%.*]] = getelementptr i32, ptr [[Y_PTR]], i64 [[IV]] |
| ; CHECK-NEXT: [[Y:%.*]] = load i32, ptr [[Y_GEP]], align 4 |
| ; CHECK-NEXT: [[C1:%.*]] = icmp eq i64 [[X]], 0 |
| ; CHECK-NEXT: br i1 [[C1]], label %[[THEN_4:.*]], label %[[THEN_1:.*]] |
| ; CHECK: [[THEN_1]]: |
| ; CHECK-NEXT: [[AND32831:%.*]] = and i32 [[Y]], 1 |
| ; CHECK-NEXT: store i64 0, ptr [[DST_1]], align 8 |
| ; CHECK-NEXT: [[C_2:%.*]] = icmp eq i32 [[Y]], 0 |
| ; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[C_4]], i1 [[C_3]], i1 false |
| ; CHECK-NEXT: br i1 [[OR_COND]], label %[[THEN_2:.*]], label %[[ELSE_1:.*]] |
| ; CHECK: [[ELSE_1]]: |
| ; CHECK-NEXT: store i64 0, ptr [[DST_3]], align 8 |
| ; CHECK-NEXT: br label %[[THEN_2]] |
| ; CHECK: [[THEN_2]]: |
| ; CHECK-NEXT: br i1 [[C_3]], label %[[THEN_3:.*]], label %[[LOOP_LATCH]] |
| ; CHECK: [[THEN_3]]: |
| ; CHECK-NEXT: br i1 [[C_4]], label %[[THEN_5:.*]], label %[[ELSE_2:.*]] |
| ; CHECK: [[THEN_4]]: |
| ; CHECK-NEXT: call void @llvm.assume(i1 [[C_4]]) |
| ; CHECK-NEXT: br label %[[THEN_5]] |
| ; CHECK: [[THEN_5]]: |
| ; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ 1, %[[THEN_4]] ], [ 0, %[[THEN_3]] ] |
| ; CHECK-NEXT: store i64 [[TMP0]], ptr [[DST_2]], align 8 |
| ; CHECK-NEXT: br label %[[ELSE_2]] |
| ; CHECK: [[ELSE_2]]: |
| ; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[SRC]], align 8 |
| ; CHECK-NEXT: store i64 [[L]], ptr [[DST]], align 8 |
| ; CHECK-NEXT: br label %[[LOOP_LATCH]] |
| ; CHECK: [[LOOP_LATCH]]: |
| ; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 64 |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP22:![0-9]+]] |
| ; CHECK: [[EXIT]]: |
| ; CHECK-NEXT: ret void |
| ; |
| entry: |
| br label %loop.header |
| |
| loop.header: |
| %iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ] |
| %x.gep = getelementptr i64, ptr %x.ptr, i64 %iv |
| %x = load i64, ptr %x.gep |
| %y.gep = getelementptr i32, ptr %y.ptr, i64 %iv |
| %y = load i32, ptr %y.gep |
| %c1 = icmp eq i64 %x, 0 |
| br i1 %c1, label %then.4, label %then.1 |
| |
| then.1: |
| %and32831 = and i32 %y, 1 |
| store i64 0, ptr %dst.1, align 8 |
| %c.2 = icmp eq i32 %y, 0 |
| %or.cond = select i1 %c.4, i1 %c.3, i1 false |
| br i1 %or.cond, label %then.2, label %else.1 |
| |
| else.1: ; preds = %then.1 |
| store i64 0, ptr %dst.3, align 8 |
| br label %then.2 |
| |
| then.2: |
| br i1 %c.3, label %then.3, label %loop.latch |
| |
| then.3: |
| br i1 %c.4, label %then.5, label %else.2 |
| |
| then.4: |
| call void @llvm.assume(i1 %c.4) |
| br label %then.5 |
| |
| then.5: |
| %1 = phi i64 [ 1, %then.4 ], [ 0, %then.3 ] |
| store i64 %1, ptr %dst.2, align 8 |
| br label %else.2 |
| |
| else.2: |
| %l = load i64, ptr %src, align 8 |
| store i64 %l, ptr %dst, align 8 |
| br label %loop.latch |
| |
| loop.latch: |
| %iv.next = add i64 %iv, 1 |
| %ec = icmp eq i64 %iv, 64 |
| br i1 %ec, label %exit, label %loop.header |
| |
| exit: |
| ret void |
| } |
| |
| declare void @llvm.assume(i1 noundef) |
| declare i64 @llvm.umin.i64(i64, i64) |
| ;. |
| ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} |
| ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} |
| ; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} |
| ; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]} |
| ; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]} |
| ; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]} |
| ; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]} |
| ; CHECK: [[META7]] = !{[[META8:![0-9]+]]} |
| ; CHECK: [[META8]] = distinct !{[[META8]], [[META9:![0-9]+]]} |
| ; CHECK: [[META9]] = distinct !{[[META9]], !"LVerDomain"} |
| ; CHECK: [[META10]] = !{[[META11:![0-9]+]], [[META12:![0-9]+]], [[META13:![0-9]+]], [[META14:![0-9]+]]} |
| ; CHECK: [[META11]] = distinct !{[[META11]], [[META9]]} |
| ; CHECK: [[META12]] = distinct !{[[META12]], [[META9]]} |
| ; CHECK: [[META13]] = distinct !{[[META13]], [[META9]]} |
| ; CHECK: [[META14]] = distinct !{[[META14]], [[META9]]} |
| ; CHECK: [[META15]] = !{[[META11]]} |
| ; CHECK: [[META16]] = !{[[META12]], [[META13]], [[META14]]} |
| ; CHECK: [[META17]] = !{[[META12]]} |
| ; CHECK: [[META18]] = !{[[META13]], [[META14]]} |
| ; CHECK: [[META19]] = !{[[META14]]} |
| ; CHECK: [[META20]] = !{[[META13]]} |
| ; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META1]], [[META2]]} |
| ; CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META1]]} |
| ;. |
