| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 6 |
| ; RUN: opt < %s -passes=loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -S | FileCheck %s |
| |
| target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" |
| target triple = "x86_64-apple-macosx10.8.0" |
| |
| @c = common global [2048 x i32] zeroinitializer, align 16 |
| @b = common global [2048 x i32] zeroinitializer, align 16 |
| @d = common global [2048 x i32] zeroinitializer, align 16 |
| @a = common global [2048 x i32] zeroinitializer, align 16 |
| |
| ; The program below gathers and scatters data. We better not vectorize it. |
| define void @cost_model_1() noinline { |
| ; CHECK-LABEL: define void @cost_model_1( |
| ; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; CHECK: [[FOR_BODY]]: |
| ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ] |
| ; CHECK-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDVARS_IV]], 1 |
| ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 [[TMP0]] |
| ; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 8 |
| ; CHECK-NEXT: [[IDXPROM1:%.*]] = sext i32 [[TMP1]] to i64 |
| ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[IDXPROM1]] |
| ; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 |
| ; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds [2048 x i32], ptr @d, i64 0, i64 [[INDVARS_IV]] |
| ; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4 |
| ; CHECK-NEXT: [[IDXPROM5:%.*]] = sext i32 [[TMP3]] to i64 |
| ; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 [[IDXPROM5]] |
| ; CHECK-NEXT: store i32 [[TMP2]], ptr [[ARRAYIDX6]], align 4 |
| ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1 |
| ; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 |
| ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256 |
| ; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]] |
| ; CHECK: [[FOR_END]]: |
| ; CHECK-NEXT: ret void |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] |
| %0 = shl nsw i64 %indvars.iv, 1 |
| %arrayidx = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 %0 |
| %1 = load i32, ptr %arrayidx, align 8 |
| %idxprom1 = sext i32 %1 to i64 |
| %arrayidx2 = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 %idxprom1 |
| %2 = load i32, ptr %arrayidx2, align 4 |
| %arrayidx4 = getelementptr inbounds [2048 x i32], ptr @d, i64 0, i64 %indvars.iv |
| %3 = load i32, ptr %arrayidx4, align 4 |
| %idxprom5 = sext i32 %3 to i64 |
| %arrayidx6 = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 %idxprom5 |
| store i32 %2, ptr %arrayidx6, align 4 |
| %indvars.iv.next = add i64 %indvars.iv, 1 |
| %lftr.wideiv = trunc i64 %indvars.iv.next to i32 |
| %exitcond = icmp eq i32 %lftr.wideiv, 256 |
| br i1 %exitcond, label %for.end, label %for.body |
| |
| for.end: |
| ret void |
| } |
| |
| ; This function uses a stride that is generally too big to benefit from vectorization without |
| ; really good support for a gather load. But if we don't vectorize the pointer induction, |
| ; then we don't need to extract the pointers out of vector of pointers, |
| ; and the vectorization becomes profitable. |
| |
| define float @PR27826(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %n) { |
| ; CHECK-LABEL: define float @PR27826( |
| ; CHECK-SAME: ptr readonly captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i32 [[N:%.*]]) #[[ATTR1:[0-9]+]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[N]], 0 |
| ; CHECK-NEXT: br i1 [[CMP]], label %[[ITER_CHECK:.*]], [[FOR_END:label %.*]] |
| ; CHECK: [[ITER_CHECK]]: |
| ; CHECK-NEXT: [[T0:%.*]] = sext i32 [[N]] to i64 |
| ; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[T0]], -1 |
| ; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 5 |
| ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 8 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP2]], 32 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 31 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] |
| ; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 5 |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP232:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP233:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP234:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP235:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[INDEX]], 5 |
| ; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP4]], 32 |
| ; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP4]], 64 |
| ; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP4]], 96 |
| ; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP4]], 128 |
| ; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP4]], 160 |
| ; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[TMP4]], 192 |
| ; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP4]], 224 |
| ; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP4]], 256 |
| ; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP4]], 288 |
| ; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP4]], 320 |
| ; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[TMP4]], 352 |
| ; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[TMP4]], 384 |
| ; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[TMP4]], 416 |
| ; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[TMP4]], 448 |
| ; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP4]], 480 |
| ; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[TMP4]], 512 |
| ; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[TMP4]], 544 |
| ; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[TMP4]], 576 |
| ; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP4]], 608 |
| ; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP4]], 640 |
| ; CHECK-NEXT: [[TMP25:%.*]] = add i64 [[TMP4]], 672 |
| ; CHECK-NEXT: [[TMP26:%.*]] = add i64 [[TMP4]], 704 |
| ; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP4]], 736 |
| ; CHECK-NEXT: [[TMP28:%.*]] = add i64 [[TMP4]], 768 |
| ; CHECK-NEXT: [[TMP29:%.*]] = add i64 [[TMP4]], 800 |
| ; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[TMP4]], 832 |
| ; CHECK-NEXT: [[TMP31:%.*]] = add i64 [[TMP4]], 864 |
| ; CHECK-NEXT: [[TMP32:%.*]] = add i64 [[TMP4]], 896 |
| ; CHECK-NEXT: [[TMP33:%.*]] = add i64 [[TMP4]], 928 |
| ; CHECK-NEXT: [[TMP34:%.*]] = add i64 [[TMP4]], 960 |
| ; CHECK-NEXT: [[TMP35:%.*]] = add i64 [[TMP4]], 992 |
| ; CHECK-NEXT: [[TMP36:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP4]] |
| ; CHECK-NEXT: [[TMP37:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP5]] |
| ; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP6]] |
| ; CHECK-NEXT: [[TMP39:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP7]] |
| ; CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]] |
| ; CHECK-NEXT: [[TMP41:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]] |
| ; CHECK-NEXT: [[TMP42:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP10]] |
| ; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP11]] |
| ; CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP12]] |
| ; CHECK-NEXT: [[TMP45:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP13]] |
| ; CHECK-NEXT: [[TMP46:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP14]] |
| ; CHECK-NEXT: [[TMP47:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]] |
| ; CHECK-NEXT: [[TMP48:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP16]] |
| ; CHECK-NEXT: [[TMP49:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP17]] |
| ; CHECK-NEXT: [[TMP50:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP18]] |
| ; CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP19]] |
| ; CHECK-NEXT: [[TMP52:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP20]] |
| ; CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP21]] |
| ; CHECK-NEXT: [[TMP54:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP22]] |
| ; CHECK-NEXT: [[TMP55:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP23]] |
| ; CHECK-NEXT: [[TMP56:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP24]] |
| ; CHECK-NEXT: [[TMP57:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP25]] |
| ; CHECK-NEXT: [[TMP58:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP26]] |
| ; CHECK-NEXT: [[TMP59:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP27]] |
| ; CHECK-NEXT: [[TMP60:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP28]] |
| ; CHECK-NEXT: [[TMP61:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP29]] |
| ; CHECK-NEXT: [[TMP62:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP30]] |
| ; CHECK-NEXT: [[TMP63:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP31]] |
| ; CHECK-NEXT: [[TMP64:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP32]] |
| ; CHECK-NEXT: [[TMP65:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP33]] |
| ; CHECK-NEXT: [[TMP66:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP34]] |
| ; CHECK-NEXT: [[TMP67:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP35]] |
| ; CHECK-NEXT: [[TMP68:%.*]] = load float, ptr [[TMP36]], align 4 |
| ; CHECK-NEXT: [[TMP69:%.*]] = load float, ptr [[TMP37]], align 4 |
| ; CHECK-NEXT: [[TMP70:%.*]] = load float, ptr [[TMP38]], align 4 |
| ; CHECK-NEXT: [[TMP71:%.*]] = load float, ptr [[TMP39]], align 4 |
| ; CHECK-NEXT: [[TMP72:%.*]] = load float, ptr [[TMP40]], align 4 |
| ; CHECK-NEXT: [[TMP73:%.*]] = load float, ptr [[TMP41]], align 4 |
| ; CHECK-NEXT: [[TMP74:%.*]] = load float, ptr [[TMP42]], align 4 |
| ; CHECK-NEXT: [[TMP75:%.*]] = load float, ptr [[TMP43]], align 4 |
| ; CHECK-NEXT: [[TMP76:%.*]] = insertelement <8 x float> poison, float [[TMP68]], i64 0 |
| ; CHECK-NEXT: [[TMP77:%.*]] = insertelement <8 x float> [[TMP76]], float [[TMP69]], i64 1 |
| ; CHECK-NEXT: [[TMP78:%.*]] = insertelement <8 x float> [[TMP77]], float [[TMP70]], i64 2 |
| ; CHECK-NEXT: [[TMP79:%.*]] = insertelement <8 x float> [[TMP78]], float [[TMP71]], i64 3 |
| ; CHECK-NEXT: [[TMP80:%.*]] = insertelement <8 x float> [[TMP79]], float [[TMP72]], i64 4 |
| ; CHECK-NEXT: [[TMP81:%.*]] = insertelement <8 x float> [[TMP80]], float [[TMP73]], i64 5 |
| ; CHECK-NEXT: [[TMP82:%.*]] = insertelement <8 x float> [[TMP81]], float [[TMP74]], i64 6 |
| ; CHECK-NEXT: [[TMP83:%.*]] = insertelement <8 x float> [[TMP82]], float [[TMP75]], i64 7 |
| ; CHECK-NEXT: [[TMP84:%.*]] = load float, ptr [[TMP44]], align 4 |
| ; CHECK-NEXT: [[TMP85:%.*]] = load float, ptr [[TMP45]], align 4 |
| ; CHECK-NEXT: [[TMP86:%.*]] = load float, ptr [[TMP46]], align 4 |
| ; CHECK-NEXT: [[TMP87:%.*]] = load float, ptr [[TMP47]], align 4 |
| ; CHECK-NEXT: [[TMP88:%.*]] = load float, ptr [[TMP48]], align 4 |
| ; CHECK-NEXT: [[TMP89:%.*]] = load float, ptr [[TMP49]], align 4 |
| ; CHECK-NEXT: [[TMP90:%.*]] = load float, ptr [[TMP50]], align 4 |
| ; CHECK-NEXT: [[TMP91:%.*]] = load float, ptr [[TMP51]], align 4 |
| ; CHECK-NEXT: [[TMP92:%.*]] = insertelement <8 x float> poison, float [[TMP84]], i64 0 |
| ; CHECK-NEXT: [[TMP93:%.*]] = insertelement <8 x float> [[TMP92]], float [[TMP85]], i64 1 |
| ; CHECK-NEXT: [[TMP94:%.*]] = insertelement <8 x float> [[TMP93]], float [[TMP86]], i64 2 |
| ; CHECK-NEXT: [[TMP95:%.*]] = insertelement <8 x float> [[TMP94]], float [[TMP87]], i64 3 |
| ; CHECK-NEXT: [[TMP96:%.*]] = insertelement <8 x float> [[TMP95]], float [[TMP88]], i64 4 |
| ; CHECK-NEXT: [[TMP97:%.*]] = insertelement <8 x float> [[TMP96]], float [[TMP89]], i64 5 |
| ; CHECK-NEXT: [[TMP98:%.*]] = insertelement <8 x float> [[TMP97]], float [[TMP90]], i64 6 |
| ; CHECK-NEXT: [[TMP99:%.*]] = insertelement <8 x float> [[TMP98]], float [[TMP91]], i64 7 |
| ; CHECK-NEXT: [[TMP100:%.*]] = load float, ptr [[TMP52]], align 4 |
| ; CHECK-NEXT: [[TMP101:%.*]] = load float, ptr [[TMP53]], align 4 |
| ; CHECK-NEXT: [[TMP102:%.*]] = load float, ptr [[TMP54]], align 4 |
| ; CHECK-NEXT: [[TMP103:%.*]] = load float, ptr [[TMP55]], align 4 |
| ; CHECK-NEXT: [[TMP104:%.*]] = load float, ptr [[TMP56]], align 4 |
| ; CHECK-NEXT: [[TMP105:%.*]] = load float, ptr [[TMP57]], align 4 |
| ; CHECK-NEXT: [[TMP106:%.*]] = load float, ptr [[TMP58]], align 4 |
| ; CHECK-NEXT: [[TMP107:%.*]] = load float, ptr [[TMP59]], align 4 |
| ; CHECK-NEXT: [[TMP108:%.*]] = insertelement <8 x float> poison, float [[TMP100]], i64 0 |
| ; CHECK-NEXT: [[TMP109:%.*]] = insertelement <8 x float> [[TMP108]], float [[TMP101]], i64 1 |
| ; CHECK-NEXT: [[TMP110:%.*]] = insertelement <8 x float> [[TMP109]], float [[TMP102]], i64 2 |
| ; CHECK-NEXT: [[TMP111:%.*]] = insertelement <8 x float> [[TMP110]], float [[TMP103]], i64 3 |
| ; CHECK-NEXT: [[TMP112:%.*]] = insertelement <8 x float> [[TMP111]], float [[TMP104]], i64 4 |
| ; CHECK-NEXT: [[TMP113:%.*]] = insertelement <8 x float> [[TMP112]], float [[TMP105]], i64 5 |
| ; CHECK-NEXT: [[TMP114:%.*]] = insertelement <8 x float> [[TMP113]], float [[TMP106]], i64 6 |
| ; CHECK-NEXT: [[TMP115:%.*]] = insertelement <8 x float> [[TMP114]], float [[TMP107]], i64 7 |
| ; CHECK-NEXT: [[TMP116:%.*]] = load float, ptr [[TMP60]], align 4 |
| ; CHECK-NEXT: [[TMP117:%.*]] = load float, ptr [[TMP61]], align 4 |
| ; CHECK-NEXT: [[TMP118:%.*]] = load float, ptr [[TMP62]], align 4 |
| ; CHECK-NEXT: [[TMP119:%.*]] = load float, ptr [[TMP63]], align 4 |
| ; CHECK-NEXT: [[TMP120:%.*]] = load float, ptr [[TMP64]], align 4 |
| ; CHECK-NEXT: [[TMP121:%.*]] = load float, ptr [[TMP65]], align 4 |
| ; CHECK-NEXT: [[TMP122:%.*]] = load float, ptr [[TMP66]], align 4 |
| ; CHECK-NEXT: [[TMP123:%.*]] = load float, ptr [[TMP67]], align 4 |
| ; CHECK-NEXT: [[TMP124:%.*]] = insertelement <8 x float> poison, float [[TMP116]], i64 0 |
| ; CHECK-NEXT: [[TMP125:%.*]] = insertelement <8 x float> [[TMP124]], float [[TMP117]], i64 1 |
| ; CHECK-NEXT: [[TMP126:%.*]] = insertelement <8 x float> [[TMP125]], float [[TMP118]], i64 2 |
| ; CHECK-NEXT: [[TMP127:%.*]] = insertelement <8 x float> [[TMP126]], float [[TMP119]], i64 3 |
| ; CHECK-NEXT: [[TMP128:%.*]] = insertelement <8 x float> [[TMP127]], float [[TMP120]], i64 4 |
| ; CHECK-NEXT: [[TMP129:%.*]] = insertelement <8 x float> [[TMP128]], float [[TMP121]], i64 5 |
| ; CHECK-NEXT: [[TMP130:%.*]] = insertelement <8 x float> [[TMP129]], float [[TMP122]], i64 6 |
| ; CHECK-NEXT: [[TMP131:%.*]] = insertelement <8 x float> [[TMP130]], float [[TMP123]], i64 7 |
| ; CHECK-NEXT: [[TMP132:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP4]] |
| ; CHECK-NEXT: [[TMP133:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP5]] |
| ; CHECK-NEXT: [[TMP134:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP6]] |
| ; CHECK-NEXT: [[TMP135:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP7]] |
| ; CHECK-NEXT: [[TMP136:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP8]] |
| ; CHECK-NEXT: [[TMP137:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP9]] |
| ; CHECK-NEXT: [[TMP138:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP10]] |
| ; CHECK-NEXT: [[TMP139:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP11]] |
| ; CHECK-NEXT: [[TMP140:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP12]] |
| ; CHECK-NEXT: [[TMP141:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP13]] |
| ; CHECK-NEXT: [[TMP142:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP14]] |
| ; CHECK-NEXT: [[TMP143:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP15]] |
| ; CHECK-NEXT: [[TMP144:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP16]] |
| ; CHECK-NEXT: [[TMP145:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP17]] |
| ; CHECK-NEXT: [[TMP146:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP18]] |
| ; CHECK-NEXT: [[TMP147:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP19]] |
| ; CHECK-NEXT: [[TMP148:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP20]] |
| ; CHECK-NEXT: [[TMP149:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP21]] |
| ; CHECK-NEXT: [[TMP150:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP22]] |
| ; CHECK-NEXT: [[TMP151:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP23]] |
| ; CHECK-NEXT: [[TMP152:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP24]] |
| ; CHECK-NEXT: [[TMP153:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP25]] |
| ; CHECK-NEXT: [[TMP154:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP26]] |
| ; CHECK-NEXT: [[TMP155:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP27]] |
| ; CHECK-NEXT: [[TMP156:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP28]] |
| ; CHECK-NEXT: [[TMP157:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP29]] |
| ; CHECK-NEXT: [[TMP158:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP30]] |
| ; CHECK-NEXT: [[TMP159:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP31]] |
| ; CHECK-NEXT: [[TMP160:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP32]] |
| ; CHECK-NEXT: [[TMP161:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP33]] |
| ; CHECK-NEXT: [[TMP162:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP34]] |
| ; CHECK-NEXT: [[TMP163:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP35]] |
| ; CHECK-NEXT: [[TMP164:%.*]] = load float, ptr [[TMP132]], align 4 |
| ; CHECK-NEXT: [[TMP165:%.*]] = load float, ptr [[TMP133]], align 4 |
| ; CHECK-NEXT: [[TMP166:%.*]] = load float, ptr [[TMP134]], align 4 |
| ; CHECK-NEXT: [[TMP167:%.*]] = load float, ptr [[TMP135]], align 4 |
| ; CHECK-NEXT: [[TMP168:%.*]] = load float, ptr [[TMP136]], align 4 |
| ; CHECK-NEXT: [[TMP169:%.*]] = load float, ptr [[TMP137]], align 4 |
| ; CHECK-NEXT: [[TMP170:%.*]] = load float, ptr [[TMP138]], align 4 |
| ; CHECK-NEXT: [[TMP171:%.*]] = load float, ptr [[TMP139]], align 4 |
| ; CHECK-NEXT: [[TMP172:%.*]] = insertelement <8 x float> poison, float [[TMP164]], i64 0 |
| ; CHECK-NEXT: [[TMP173:%.*]] = insertelement <8 x float> [[TMP172]], float [[TMP165]], i64 1 |
| ; CHECK-NEXT: [[TMP174:%.*]] = insertelement <8 x float> [[TMP173]], float [[TMP166]], i64 2 |
| ; CHECK-NEXT: [[TMP175:%.*]] = insertelement <8 x float> [[TMP174]], float [[TMP167]], i64 3 |
| ; CHECK-NEXT: [[TMP176:%.*]] = insertelement <8 x float> [[TMP175]], float [[TMP168]], i64 4 |
| ; CHECK-NEXT: [[TMP177:%.*]] = insertelement <8 x float> [[TMP176]], float [[TMP169]], i64 5 |
| ; CHECK-NEXT: [[TMP178:%.*]] = insertelement <8 x float> [[TMP177]], float [[TMP170]], i64 6 |
| ; CHECK-NEXT: [[TMP179:%.*]] = insertelement <8 x float> [[TMP178]], float [[TMP171]], i64 7 |
| ; CHECK-NEXT: [[TMP180:%.*]] = load float, ptr [[TMP140]], align 4 |
| ; CHECK-NEXT: [[TMP181:%.*]] = load float, ptr [[TMP141]], align 4 |
| ; CHECK-NEXT: [[TMP182:%.*]] = load float, ptr [[TMP142]], align 4 |
| ; CHECK-NEXT: [[TMP183:%.*]] = load float, ptr [[TMP143]], align 4 |
| ; CHECK-NEXT: [[TMP184:%.*]] = load float, ptr [[TMP144]], align 4 |
| ; CHECK-NEXT: [[TMP185:%.*]] = load float, ptr [[TMP145]], align 4 |
| ; CHECK-NEXT: [[TMP186:%.*]] = load float, ptr [[TMP146]], align 4 |
| ; CHECK-NEXT: [[TMP187:%.*]] = load float, ptr [[TMP147]], align 4 |
| ; CHECK-NEXT: [[TMP188:%.*]] = insertelement <8 x float> poison, float [[TMP180]], i64 0 |
| ; CHECK-NEXT: [[TMP189:%.*]] = insertelement <8 x float> [[TMP188]], float [[TMP181]], i64 1 |
| ; CHECK-NEXT: [[TMP190:%.*]] = insertelement <8 x float> [[TMP189]], float [[TMP182]], i64 2 |
| ; CHECK-NEXT: [[TMP191:%.*]] = insertelement <8 x float> [[TMP190]], float [[TMP183]], i64 3 |
| ; CHECK-NEXT: [[TMP192:%.*]] = insertelement <8 x float> [[TMP191]], float [[TMP184]], i64 4 |
| ; CHECK-NEXT: [[TMP193:%.*]] = insertelement <8 x float> [[TMP192]], float [[TMP185]], i64 5 |
| ; CHECK-NEXT: [[TMP194:%.*]] = insertelement <8 x float> [[TMP193]], float [[TMP186]], i64 6 |
| ; CHECK-NEXT: [[TMP195:%.*]] = insertelement <8 x float> [[TMP194]], float [[TMP187]], i64 7 |
| ; CHECK-NEXT: [[TMP196:%.*]] = load float, ptr [[TMP148]], align 4 |
| ; CHECK-NEXT: [[TMP197:%.*]] = load float, ptr [[TMP149]], align 4 |
| ; CHECK-NEXT: [[TMP198:%.*]] = load float, ptr [[TMP150]], align 4 |
| ; CHECK-NEXT: [[TMP199:%.*]] = load float, ptr [[TMP151]], align 4 |
| ; CHECK-NEXT: [[TMP200:%.*]] = load float, ptr [[TMP152]], align 4 |
| ; CHECK-NEXT: [[TMP201:%.*]] = load float, ptr [[TMP153]], align 4 |
| ; CHECK-NEXT: [[TMP202:%.*]] = load float, ptr [[TMP154]], align 4 |
| ; CHECK-NEXT: [[TMP203:%.*]] = load float, ptr [[TMP155]], align 4 |
| ; CHECK-NEXT: [[TMP204:%.*]] = insertelement <8 x float> poison, float [[TMP196]], i64 0 |
| ; CHECK-NEXT: [[TMP205:%.*]] = insertelement <8 x float> [[TMP204]], float [[TMP197]], i64 1 |
| ; CHECK-NEXT: [[TMP206:%.*]] = insertelement <8 x float> [[TMP205]], float [[TMP198]], i64 2 |
| ; CHECK-NEXT: [[TMP207:%.*]] = insertelement <8 x float> [[TMP206]], float [[TMP199]], i64 3 |
| ; CHECK-NEXT: [[TMP208:%.*]] = insertelement <8 x float> [[TMP207]], float [[TMP200]], i64 4 |
| ; CHECK-NEXT: [[TMP209:%.*]] = insertelement <8 x float> [[TMP208]], float [[TMP201]], i64 5 |
| ; CHECK-NEXT: [[TMP210:%.*]] = insertelement <8 x float> [[TMP209]], float [[TMP202]], i64 6 |
| ; CHECK-NEXT: [[TMP211:%.*]] = insertelement <8 x float> [[TMP210]], float [[TMP203]], i64 7 |
| ; CHECK-NEXT: [[TMP212:%.*]] = load float, ptr [[TMP156]], align 4 |
| ; CHECK-NEXT: [[TMP213:%.*]] = load float, ptr [[TMP157]], align 4 |
| ; CHECK-NEXT: [[TMP214:%.*]] = load float, ptr [[TMP158]], align 4 |
| ; CHECK-NEXT: [[TMP215:%.*]] = load float, ptr [[TMP159]], align 4 |
| ; CHECK-NEXT: [[TMP216:%.*]] = load float, ptr [[TMP160]], align 4 |
| ; CHECK-NEXT: [[TMP217:%.*]] = load float, ptr [[TMP161]], align 4 |
| ; CHECK-NEXT: [[TMP218:%.*]] = load float, ptr [[TMP162]], align 4 |
| ; CHECK-NEXT: [[TMP219:%.*]] = load float, ptr [[TMP163]], align 4 |
| ; CHECK-NEXT: [[TMP220:%.*]] = insertelement <8 x float> poison, float [[TMP212]], i64 0 |
| ; CHECK-NEXT: [[TMP221:%.*]] = insertelement <8 x float> [[TMP220]], float [[TMP213]], i64 1 |
| ; CHECK-NEXT: [[TMP222:%.*]] = insertelement <8 x float> [[TMP221]], float [[TMP214]], i64 2 |
| ; CHECK-NEXT: [[TMP223:%.*]] = insertelement <8 x float> [[TMP222]], float [[TMP215]], i64 3 |
| ; CHECK-NEXT: [[TMP224:%.*]] = insertelement <8 x float> [[TMP223]], float [[TMP216]], i64 4 |
| ; CHECK-NEXT: [[TMP225:%.*]] = insertelement <8 x float> [[TMP224]], float [[TMP217]], i64 5 |
| ; CHECK-NEXT: [[TMP226:%.*]] = insertelement <8 x float> [[TMP225]], float [[TMP218]], i64 6 |
| ; CHECK-NEXT: [[TMP227:%.*]] = insertelement <8 x float> [[TMP226]], float [[TMP219]], i64 7 |
| ; CHECK-NEXT: [[TMP228:%.*]] = fadd fast <8 x float> [[TMP83]], [[VEC_PHI]] |
| ; CHECK-NEXT: [[TMP229:%.*]] = fadd fast <8 x float> [[TMP99]], [[VEC_PHI2]] |
| ; CHECK-NEXT: [[TMP230:%.*]] = fadd fast <8 x float> [[TMP115]], [[VEC_PHI3]] |
| ; CHECK-NEXT: [[TMP231:%.*]] = fadd fast <8 x float> [[TMP131]], [[VEC_PHI4]] |
| ; CHECK-NEXT: [[TMP232]] = fadd fast <8 x float> [[TMP228]], [[TMP179]] |
| ; CHECK-NEXT: [[TMP233]] = fadd fast <8 x float> [[TMP229]], [[TMP195]] |
| ; CHECK-NEXT: [[TMP234]] = fadd fast <8 x float> [[TMP230]], [[TMP211]] |
| ; CHECK-NEXT: [[TMP235]] = fadd fast <8 x float> [[TMP231]], [[TMP227]] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32 |
| ; CHECK-NEXT: [[TMP236:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP236]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP233]], [[TMP232]] |
| ; CHECK-NEXT: [[BIN_RDX5:%.*]] = fadd fast <8 x float> [[TMP234]], [[BIN_RDX]] |
| ; CHECK-NEXT: [[BIN_RDX6:%.*]] = fadd fast <8 x float> [[TMP235]], [[BIN_RDX5]] |
| ; CHECK-NEXT: [[TMP237:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[BIN_RDX6]]) |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], [[LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8 |
| ; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]] |
| ; CHECK: [[VEC_EPILOG_PH]]: |
| ; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP237]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; CHECK-NEXT: [[N_MOD_VF7:%.*]] = and i64 [[TMP2]], 7 |
| ; CHECK-NEXT: [[N_VEC8:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF7]] |
| ; CHECK-NEXT: [[TMP238:%.*]] = shl i64 [[N_VEC8]], 5 |
| ; CHECK-NEXT: [[TMP239:%.*]] = insertelement <8 x float> zeroinitializer, float [[BC_MERGE_RDX]], i64 0 |
| ; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI10:%.*]] = phi <8 x float> [ [[TMP239]], %[[VEC_EPILOG_PH]] ], [ [[TMP297:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[TMP240:%.*]] = shl nuw i64 [[INDEX9]], 5 |
| ; CHECK-NEXT: [[TMP241:%.*]] = add i64 [[TMP240]], 32 |
| ; CHECK-NEXT: [[TMP242:%.*]] = add i64 [[TMP240]], 64 |
| ; CHECK-NEXT: [[TMP243:%.*]] = add i64 [[TMP240]], 96 |
| ; CHECK-NEXT: [[TMP272:%.*]] = add i64 [[TMP240]], 128 |
| ; CHECK-NEXT: [[TMP273:%.*]] = add i64 [[TMP240]], 160 |
| ; CHECK-NEXT: [[TMP274:%.*]] = add i64 [[TMP240]], 192 |
| ; CHECK-NEXT: [[TMP275:%.*]] = add i64 [[TMP240]], 224 |
| ; CHECK-NEXT: [[TMP244:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP240]] |
| ; CHECK-NEXT: [[TMP245:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP241]] |
| ; CHECK-NEXT: [[TMP246:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP242]] |
| ; CHECK-NEXT: [[TMP247:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP243]] |
| ; CHECK-NEXT: [[TMP252:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP272]] |
| ; CHECK-NEXT: [[TMP253:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP273]] |
| ; CHECK-NEXT: [[TMP254:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP274]] |
| ; CHECK-NEXT: [[TMP255:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP275]] |
| ; CHECK-NEXT: [[TMP248:%.*]] = load float, ptr [[TMP244]], align 4 |
| ; CHECK-NEXT: [[TMP249:%.*]] = load float, ptr [[TMP245]], align 4 |
| ; CHECK-NEXT: [[TMP250:%.*]] = load float, ptr [[TMP246]], align 4 |
| ; CHECK-NEXT: [[TMP251:%.*]] = load float, ptr [[TMP247]], align 4 |
| ; CHECK-NEXT: [[TMP280:%.*]] = load float, ptr [[TMP252]], align 4 |
| ; CHECK-NEXT: [[TMP281:%.*]] = load float, ptr [[TMP253]], align 4 |
| ; CHECK-NEXT: [[TMP282:%.*]] = load float, ptr [[TMP254]], align 4 |
| ; CHECK-NEXT: [[TMP283:%.*]] = load float, ptr [[TMP255]], align 4 |
| ; CHECK-NEXT: [[TMP264:%.*]] = insertelement <8 x float> poison, float [[TMP248]], i64 0 |
| ; CHECK-NEXT: [[TMP265:%.*]] = insertelement <8 x float> [[TMP264]], float [[TMP249]], i64 1 |
| ; CHECK-NEXT: [[TMP266:%.*]] = insertelement <8 x float> [[TMP265]], float [[TMP250]], i64 2 |
| ; CHECK-NEXT: [[TMP267:%.*]] = insertelement <8 x float> [[TMP266]], float [[TMP251]], i64 3 |
| ; CHECK-NEXT: [[TMP268:%.*]] = insertelement <8 x float> [[TMP267]], float [[TMP280]], i64 4 |
| ; CHECK-NEXT: [[TMP269:%.*]] = insertelement <8 x float> [[TMP268]], float [[TMP281]], i64 5 |
| ; CHECK-NEXT: [[TMP298:%.*]] = insertelement <8 x float> [[TMP269]], float [[TMP282]], i64 6 |
| ; CHECK-NEXT: [[TMP271:%.*]] = insertelement <8 x float> [[TMP298]], float [[TMP283]], i64 7 |
| ; CHECK-NEXT: [[TMP256:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP240]] |
| ; CHECK-NEXT: [[TMP257:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP241]] |
| ; CHECK-NEXT: [[TMP258:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP242]] |
| ; CHECK-NEXT: [[TMP259:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP243]] |
| ; CHECK-NEXT: [[TMP276:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP272]] |
| ; CHECK-NEXT: [[TMP277:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP273]] |
| ; CHECK-NEXT: [[TMP278:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP274]] |
| ; CHECK-NEXT: [[TMP279:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP275]] |
| ; CHECK-NEXT: [[TMP260:%.*]] = load float, ptr [[TMP256]], align 4 |
| ; CHECK-NEXT: [[TMP261:%.*]] = load float, ptr [[TMP257]], align 4 |
| ; CHECK-NEXT: [[TMP262:%.*]] = load float, ptr [[TMP258]], align 4 |
| ; CHECK-NEXT: [[TMP263:%.*]] = load float, ptr [[TMP259]], align 4 |
| ; CHECK-NEXT: [[TMP284:%.*]] = load float, ptr [[TMP276]], align 4 |
| ; CHECK-NEXT: [[TMP285:%.*]] = load float, ptr [[TMP277]], align 4 |
| ; CHECK-NEXT: [[TMP286:%.*]] = load float, ptr [[TMP278]], align 4 |
| ; CHECK-NEXT: [[TMP287:%.*]] = load float, ptr [[TMP279]], align 4 |
| ; CHECK-NEXT: [[TMP288:%.*]] = insertelement <8 x float> poison, float [[TMP260]], i64 0 |
| ; CHECK-NEXT: [[TMP289:%.*]] = insertelement <8 x float> [[TMP288]], float [[TMP261]], i64 1 |
| ; CHECK-NEXT: [[TMP290:%.*]] = insertelement <8 x float> [[TMP289]], float [[TMP262]], i64 2 |
| ; CHECK-NEXT: [[TMP291:%.*]] = insertelement <8 x float> [[TMP290]], float [[TMP263]], i64 3 |
| ; CHECK-NEXT: [[TMP292:%.*]] = insertelement <8 x float> [[TMP291]], float [[TMP284]], i64 4 |
| ; CHECK-NEXT: [[TMP293:%.*]] = insertelement <8 x float> [[TMP292]], float [[TMP285]], i64 5 |
| ; CHECK-NEXT: [[TMP294:%.*]] = insertelement <8 x float> [[TMP293]], float [[TMP286]], i64 6 |
| ; CHECK-NEXT: [[TMP295:%.*]] = insertelement <8 x float> [[TMP294]], float [[TMP287]], i64 7 |
| ; CHECK-NEXT: [[TMP296:%.*]] = fadd fast <8 x float> [[TMP271]], [[VEC_PHI10]] |
| ; CHECK-NEXT: [[TMP297]] = fadd fast <8 x float> [[TMP296]], [[TMP295]] |
| ; CHECK-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX9]], 8 |
| ; CHECK-NEXT: [[TMP270:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC8]] |
| ; CHECK-NEXT: br i1 [[TMP270]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] |
| ; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[TMP299:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[TMP297]]) |
| ; CHECK-NEXT: [[CMP_N12:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC8]] |
| ; CHECK-NEXT: br i1 [[CMP_N12]], [[LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; |
| entry: |
| %cmp = icmp sgt i32 %n, 0 |
| br i1 %cmp, label %preheader, label %for.end |
| |
| preheader: |
| %t0 = sext i32 %n to i64 |
| br label %for |
| |
| for: |
| %indvars.iv = phi i64 [ 0, %preheader ], [ %indvars.iv.next, %for ] |
| %s.02 = phi float [ 0.0, %preheader ], [ %add4, %for ] |
| %arrayidx = getelementptr inbounds float, ptr %a, i64 %indvars.iv |
| %t1 = load float, ptr %arrayidx, align 4 |
| %arrayidx3 = getelementptr inbounds float, ptr %b, i64 %indvars.iv |
| %t2 = load float, ptr %arrayidx3, align 4 |
| %add = fadd fast float %t1, %s.02 |
| %add4 = fadd fast float %add, %t2 |
| %indvars.iv.next = add nuw nsw i64 %indvars.iv, 32 |
| %cmp1 = icmp slt i64 %indvars.iv.next, %t0 |
| br i1 %cmp1, label %for, label %loopexit |
| |
| loopexit: |
| %add4.lcssa = phi float [ %add4, %for ] |
| br label %for.end |
| |
| for.end: |
| %s.0.lcssa = phi float [ 0.0, %entry ], [ %add4.lcssa, %loopexit ] |
| ret float %s.0.lcssa |
| } |
| |
| define void @multi_exit(ptr %dst, ptr %src.1, ptr %src.2, i64 %A, i64 %B) #0 { |
| ; CHECK-LABEL: define void @multi_exit( |
| ; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC_1:%.*]], ptr [[SRC_2:%.*]], i64 [[A:%.*]], i64 [[B:%.*]]) #[[ATTR2:[0-9]+]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.umax.i64(i64 [[B]], i64 1) |
| ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[TMP10]], -1 |
| ; CHECK-NEXT: [[TMP1:%.*]] = freeze i64 [[TMP0]] |
| ; CHECK-NEXT: [[UMIN10:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[A]]) |
| ; CHECK-NEXT: [[TMP2:%.*]] = add nuw i64 [[UMIN10]], 1 |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], 24 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]] |
| ; CHECK: [[VECTOR_SCEVCHECK]]: |
| ; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[B]], i64 1) |
| ; CHECK-NEXT: [[TMP4:%.*]] = freeze i64 [[TMP3]] |
| ; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP4]], i64 [[A]]) |
| ; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[UMIN]] to i32 |
| ; CHECK-NEXT: [[TMP6:%.*]] = add i32 1, [[TMP5]] |
| ; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i32 [[TMP6]], 1 |
| ; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[UMIN]], 4294967295 |
| ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] |
| ; CHECK-NEXT: [[TMP11:%.*]] = icmp slt i32 [[TMP5]], 0 |
| ; CHECK-NEXT: [[TMP12:%.*]] = icmp ugt i64 [[UMIN]], 4294967295 |
| ; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]] |
| ; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP9]], [[TMP13]] |
| ; CHECK-NEXT: br i1 [[TMP14]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]] |
| ; CHECK: [[VECTOR_MEMCHECK]]: |
| ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 1 |
| ; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC_2]], i64 8 |
| ; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[B]], i64 1) |
| ; CHECK-NEXT: [[TMP16:%.*]] = freeze i64 [[TMP15]] |
| ; CHECK-NEXT: [[UMIN4:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP16]], i64 [[A]]) |
| ; CHECK-NEXT: [[TMP17:%.*]] = shl i64 [[UMIN4]], 3 |
| ; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[TMP17]], 8 |
| ; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[SRC_1]], i64 [[TMP18]] |
| ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]] |
| ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC_2]], [[SCEVGEP]] |
| ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] |
| ; CHECK-NEXT: [[BOUND06:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP5]] |
| ; CHECK-NEXT: [[BOUND17:%.*]] = icmp ult ptr [[SRC_1]], [[SCEVGEP]] |
| ; CHECK-NEXT: [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]] |
| ; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT8]] |
| ; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 3 |
| ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 |
| ; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP19]], i64 4, i64 [[N_MOD_VF]] |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP20]] |
| ; CHECK-NEXT: [[TMP21:%.*]] = trunc i64 [[N_VEC]] to i32 |
| ; CHECK-NEXT: [[TMP22:%.*]] = load i64, ptr [[SRC_2]], align 8, !alias.scope [[META6:![0-9]+]] |
| ; CHECK-NEXT: [[TMP23:%.*]] = icmp ne i64 [[TMP22]], 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[TMP23]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[TMP24:%.*]] = trunc i64 [[INDEX]] to i32 |
| ; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i64, ptr [[SRC_1]], i32 [[TMP24]] |
| ; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i64, ptr [[TMP25]], i64 2 |
| ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP26]], align 8, !alias.scope [[META9:![0-9]+]] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 |
| ; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[TMP27:%.*]] = icmp eq <2 x i64> [[WIDE_LOAD]], zeroinitializer |
| ; CHECK-NEXT: [[TMP28:%.*]] = and <2 x i1> [[BROADCAST_SPLAT]], [[TMP27]] |
| ; CHECK-NEXT: [[TMP29:%.*]] = zext <2 x i1> [[TMP28]] to <2 x i8> |
| ; CHECK-NEXT: [[TMP30:%.*]] = extractelement <2 x i8> [[TMP29]], i64 1 |
| ; CHECK-NEXT: store i8 [[TMP30]], ptr [[DST]], align 1, !alias.scope [[META12:![0-9]+]], !noalias [[META14:![0-9]+]] |
| ; CHECK-NEXT: br label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv.1.wide = phi i64 [ 0, %entry ], [ %iv.1.next.wide, %loop.latch ] |
| %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop.latch ] |
| %ec.1 = icmp ult i64 %iv.1.wide, %A |
| br i1 %ec.1, label %loop.latch, label %exit |
| |
| loop.latch: |
| %gep.src.1 = getelementptr inbounds i64, ptr %src.1, i32 %iv.1 |
| %l.1 = load i64, ptr %gep.src.1, align 8 |
| %l.2 = load i64, ptr %src.2, align 8 |
| %cmp55.us = icmp eq i64 %l.1, 0 |
| %cmp.i.us = icmp ne i64 %l.2, 0 |
| %and = and i1 %cmp.i.us, %cmp55.us |
| %ext = zext i1 %and to i8 |
| store i8 %ext, ptr %dst, align 1 |
| %iv.1.next = add i32 %iv.1, 1 |
| %iv.1.next.wide = zext i32 %iv.1.next to i64 |
| %ec.2 = icmp ult i64 %iv.1.next.wide, %B |
| br i1 %ec.2, label %loop, label %exit |
| |
| exit: |
| ret void |
| } |
| |
| define i1 @any_of_cost(ptr %start, ptr %end) #0 { |
| ; CHECK-LABEL: define i1 @any_of_cost( |
| ; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) #[[ATTR2]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[END1:%.*]] = ptrtoaddr ptr [[END]] to i64 |
| ; CHECK-NEXT: [[START2:%.*]] = ptrtoaddr ptr [[START]] to i64 |
| ; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[END1]], [[START2]] |
| ; CHECK-NEXT: [[TMP1:%.*]] = udiv i64 [[TMP0]], 40 |
| ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], 4 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 3 |
| ; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 |
| ; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i64 4, i64 [[N_MOD_VF]] |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP4]] |
| ; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[N_VEC]], 40 |
| ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP5]] |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[INDEX]], 40 |
| ; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP7]], 40 |
| ; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP7]], 80 |
| ; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[TMP7]], 120 |
| ; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP7]] |
| ; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]] |
| ; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP9]] |
| ; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP10]] |
| ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 8 |
| ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 8 |
| ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 8 |
| ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 8 |
| ; CHECK-NEXT: [[TMP15:%.*]] = load ptr, ptr [[TMP11]], align 8 |
| ; CHECK-NEXT: [[TMP16:%.*]] = load ptr, ptr [[TMP12]], align 8 |
| ; CHECK-NEXT: [[TMP17:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP15]], i64 0 |
| ; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x ptr> [[TMP17]], ptr [[TMP16]], i64 1 |
| ; CHECK-NEXT: [[TMP19:%.*]] = load ptr, ptr [[TMP13]], align 8 |
| ; CHECK-NEXT: [[TMP20:%.*]] = load ptr, ptr [[TMP14]], align 8 |
| ; CHECK-NEXT: [[TMP21:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP19]], i64 0 |
| ; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x ptr> [[TMP21]], ptr [[TMP20]], i64 1 |
| ; CHECK-NEXT: [[TMP23:%.*]] = icmp ne <2 x ptr> [[TMP18]], splat (ptr null) |
| ; CHECK-NEXT: [[TMP24:%.*]] = icmp ne <2 x ptr> [[TMP22]], splat (ptr null) |
| ; CHECK-NEXT: [[TMP25]] = or <2 x i1> [[VEC_PHI]], [[TMP23]] |
| ; CHECK-NEXT: [[TMP26]] = or <2 x i1> [[VEC_PHI3]], [[TMP24]] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 |
| ; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP27]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[BIN_RDX:%.*]] = or <2 x i1> [[TMP26]], [[TMP25]] |
| ; CHECK-NEXT: [[TMP28:%.*]] = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> [[BIN_RDX]]) |
| ; CHECK-NEXT: [[TMP29:%.*]] = freeze i1 [[TMP28]] |
| ; CHECK-NEXT: br label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %any.of = phi i1 [ false, %entry ], [ %any.of.next, %loop ] |
| %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop ] |
| %gep = getelementptr i8, ptr %ptr.iv, i64 8 |
| %l = load ptr, ptr %gep, align 8 |
| %cmp13.not.not = icmp eq ptr %l, null |
| %any.of.next = select i1 %cmp13.not.not, i1 %any.of, i1 true |
| %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 40 |
| %cmp.not = icmp eq ptr %ptr.iv, %end |
| br i1 %cmp.not, label %exit, label %loop, !llvm.loop !0 |
| |
| exit: |
| ret i1 %any.of.next |
| } |
| |
| define i64 @cost_assume(ptr %end, i64 %N) { |
| ; CHECK-LABEL: define i64 @cost_assume( |
| ; CHECK-SAME: ptr [[END:%.*]], i64 [[N:%.*]]) #[[ATTR1]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[END1:%.*]] = ptrtoaddr ptr [[END]] to i64 |
| ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[END1]], -9 |
| ; CHECK-NEXT: [[TMP1:%.*]] = udiv i64 [[TMP0]], 9 |
| ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 8 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP2]], 7 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] |
| ; CHECK-NEXT: [[TMP3:%.*]] = icmp ne i64 [[N]], 0 |
| ; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP3]]) |
| ; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP3]]) |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[TMP4]] = add <2 x i64> [[VEC_PHI]], splat (i64 1) |
| ; CHECK-NEXT: [[TMP5]] = add <2 x i64> [[VEC_PHI2]], splat (i64 1) |
| ; CHECK-NEXT: [[TMP6]] = add <2 x i64> [[VEC_PHI3]], splat (i64 1) |
| ; CHECK-NEXT: [[TMP7]] = add <2 x i64> [[VEC_PHI4]], splat (i64 1) |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 |
| ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[BIN_RDX:%.*]] = add <2 x i64> [[TMP5]], [[TMP4]] |
| ; CHECK-NEXT: [[BIN_RDX5:%.*]] = add <2 x i64> [[TMP6]], [[BIN_RDX]] |
| ; CHECK-NEXT: [[BIN_RDX6:%.*]] = add <2 x i64> [[TMP7]], [[BIN_RDX5]] |
| ; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX6]]) |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %0 = phi i64 [ 0, %entry ], [ %1, %loop ] |
| %1 = add i64 %0, 1 |
| %iv.next = add nsw i64 %iv, 1 |
| %c = icmp ne i64 %N, 0 |
| tail call void @llvm.assume(i1 %c) |
| %gep = getelementptr nusw [ 9 x i8 ], ptr null, i64 %iv.next |
| %ec = icmp eq ptr %gep, %end |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret i64 %1 |
| } |
| |
| ; Test case for https://github.com/llvm/llvm-project/issues/96294 with a stored |
| ; reduction which overwrites an earlier store. |
| define void @reduction_store(ptr noalias %src, ptr %dst, i1 %x) #2 { |
| ; CHECK-LABEL: define void @reduction_store( |
| ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr [[DST:%.*]], i1 [[X:%.*]]) #[[ATTR3:[0-9]+]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[X]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP0:%.*]] = zext <4 x i1> [[BROADCAST_SPLAT]] to <4 x i64> |
| ; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i64> [[TMP0]], splat (i64 12) |
| ; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i64> [[TMP1]] to <4 x i32> |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 0, i32 -1, i32 -1, i32 -1>, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[TMP3]] = and <4 x i32> [[VEC_PHI]], [[TMP2]] |
| ; CHECK-NEXT: [[TMP4]] = and <4 x i32> [[VEC_PHI1]], [[TMP2]] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 |
| ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 24 |
| ; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[BIN_RDX:%.*]] = and <4 x i32> [[TMP4]], [[TMP3]] |
| ; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[BIN_RDX]]) |
| ; CHECK-NEXT: store i32 [[TMP6]], ptr [[DST]], align 4 |
| ; CHECK-NEXT: br label %[[SCALAR_PH:.*]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] |
| %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] |
| %gep.src = getelementptr inbounds i32, ptr %src, i32 %iv |
| %l = load i32, ptr %gep.src |
| %l.and = and i32 %l, 3 |
| store i32 %l.and, ptr %dst, align 4 |
| %x.ext = zext i1 %x to i64 |
| %lshr = lshr i64 %x.ext, 12 |
| %t = trunc i64 %lshr to i32 |
| %red.next = and i32 %red, %t |
| store i32 %red.next, ptr %dst, align 4 |
| %iv.next = add i32 %iv, 1 |
| %ec = icmp eq i32 %iv, 29 |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| ; Test case for https://github.com/llvm/llvm-project/issues/105722. |
| define i64 @live_in_known_1_via_scev() { |
| ; CHECK-LABEL: define i64 @live_in_known_1_via_scev( |
| ; CHECK-SAME: ) #[[ATTR1]] { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: [[SEL:%.*]] = select i1 false, i32 3, i32 0 |
| ; CHECK-NEXT: br label %[[PH:.*]] |
| ; CHECK: [[PH]]: |
| ; CHECK-NEXT: [[P:%.*]] = phi i32 [ 1, %[[ENTRY]] ] |
| ; CHECK-NEXT: [[N:%.*]] = add nuw nsw i32 [[SEL]], 6 |
| ; CHECK-NEXT: [[P_EXT:%.*]] = zext nneg i32 [[P]] to i64 |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[PH]] ], [ [[IV_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[RED:%.*]] = phi i64 [ 3, %[[PH]] ], [ [[RED_MUL:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[RED_MUL]] = mul nsw i64 [[RED]], [[P_EXT]] |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]] |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[VECTOR_BODY]] |
| ; CHECK: [[EXIT]]: |
| ; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ [[RED_MUL]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: ret i64 [[TMP1]] |
| ; |
| entry: |
| %sel = select i1 false, i32 3, i32 0 |
| br label %ph |
| |
| ph: |
| %p = phi i32 [ 1, %entry ] |
| %N = add nuw nsw i32 %sel, 6 |
| %p.ext = zext nneg i32 %p to i64 |
| br label %loop |
| |
| loop: |
| %iv = phi i32 [ 0, %ph ], [ %iv.next, %loop ] |
| %red = phi i64 [ 3, %ph ], [ %red.mul, %loop ] |
| %red.mul = mul nsw i64 %red, %p.ext |
| %iv.next = add nuw nsw i32 %iv, 1 |
| %ec = icmp eq i32 %iv.next, %N |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| %res = phi i64 [ %red.mul, %loop ] |
| ret i64 %res |
| } |
| |
| ; Test case for https://github.com/llvm/llvm-project/issues/107501. |
| define i64 @cost_loop_invariant_recipes(i1 %x, i64 %y) { |
| ; CHECK-LABEL: define i64 @cost_loop_invariant_recipes( |
| ; CHECK-SAME: i1 [[X:%.*]], i64 [[Y:%.*]]) #[[ATTR1]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[Y]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[X]], true |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i1> poison, i1 [[TMP0]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT1]], <2 x i1> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i1> [[BROADCAST_SPLAT2]] to <2 x i64> |
| ; CHECK-NEXT: [[TMP2:%.*]] = shl <2 x i64> [[BROADCAST_SPLAT]], [[TMP1]] |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[TMP3:%.*]] = mul <2 x i64> [[TMP2]], splat (i64 1) |
| ; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.mul.v2i64(<2 x i64> [[TMP3]]) |
| ; CHECK-NEXT: br label %[[EXIT:.*]] |
| ; CHECK: [[EXIT]]: |
| ; CHECK-NEXT: ret i64 [[TMP4]] |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next.i.i.i, %loop ] |
| %red = phi i64 [ 1, %entry ], [ %red.mul, %loop ] |
| %not.x = xor i1 %x, true |
| %ext = zext i1 %not.x to i64 |
| %shl = shl i64 %y, %ext |
| %red.mul = mul i64 %shl, %red |
| %iv.next.i.i.i = add i64 %iv, 1 |
| %ec = icmp eq i64 %iv, 1 |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret i64 %red.mul |
| } |
| |
| ; Test case for https://github.com/llvm/llvm-project/issues/113526. |
| define i32 @narrowed_reduction(ptr %a, i1 %cmp) #0 { |
| ; |
| ; CHECK-LABEL: define i32 @narrowed_reduction( |
| ; CHECK-SAME: ptr [[A:%.*]], i1 [[CMP:%.*]]) #[[ATTR2]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[CONV:%.*]] = zext i1 [[CMP]] to i32 |
| ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[CONV]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[TMP0:%.*]] = and <4 x i32> [[VEC_PHI]], splat (i32 1) |
| ; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[VEC_PHI1]], splat (i32 1) |
| ; CHECK-NEXT: [[TMP2:%.*]] = or <4 x i32> [[TMP0]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: [[TMP3:%.*]] = or <4 x i32> [[TMP1]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i32> [[TMP2]] to <4 x i1> |
| ; CHECK-NEXT: [[TMP5:%.*]] = trunc <4 x i32> [[TMP3]] to <4 x i1> |
| ; CHECK-NEXT: [[TMP6]] = zext <4 x i1> [[TMP4]] to <4 x i32> |
| ; CHECK-NEXT: [[TMP7]] = zext <4 x i1> [[TMP5]] to <4 x i32> |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 |
| ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 48 |
| ; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[BIN_RDX:%.*]] = or <4 x i1> [[TMP5]], [[TMP4]] |
| ; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX]]) |
| ; CHECK-NEXT: [[TMP10:%.*]] = zext i1 [[TMP9]] to i32 |
| ; CHECK-NEXT: br label %[[EXIT:.*]] |
| ; CHECK: [[EXIT]]: |
| ; CHECK-NEXT: ret i32 [[TMP10]] |
| ; |
| entry: |
| %conv = zext i1 %cmp to i32 |
| br label %loop |
| |
| loop: |
| %iv = phi i32 [ 1, %entry ], [ %inc, %loop ] |
| %or13 = phi i32 [ 0, %entry ], [ %or, %loop ] |
| %and = and i32 %or13, 1 |
| %or = or i32 %and, %conv |
| %inc = add i32 %iv, 1 |
| %ec = icmp eq i32 %iv, 48 |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret i32 %or |
| } |
| |
| ; Check if the vplan-based cost model select same VF to the legacy cost model. |
| ; Reduced from: https://github.com/llvm/llvm-project/issues/115744#issuecomment-2670479463 |
| define i32 @g(i64 %n) { |
| ; CHECK-LABEL: define i32 @g( |
| ; CHECK-SAME: i64 [[N:%.*]]) #[[ATTR1]] { |
| ; CHECK-NEXT: [[ITER_CHECK:.*:]] |
| ; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[N]] to i32 |
| ; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 1 |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 4 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]] |
| ; CHECK: [[VECTOR_SCEVCHECK]]: |
| ; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[N]], 4294967295 |
| ; CHECK-NEXT: br i1 [[TMP2]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] |
| ; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i32 [[TMP1]], 32 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i32 [[TMP1]], 31 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]] |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[N]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <8 x i32> [[VEC_IND]], splat (i32 8) |
| ; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <8 x i32> [[STEP_ADD]], splat (i32 8) |
| ; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <8 x i32> [[STEP_ADD_2]], splat (i32 8) |
| ; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i32> [[VEC_IND]] to <8 x i64> |
| ; CHECK-NEXT: [[TMP4:%.*]] = zext <8 x i32> [[STEP_ADD]] to <8 x i64> |
| ; CHECK-NEXT: [[TMP5:%.*]] = zext <8 x i32> [[STEP_ADD_2]] to <8 x i64> |
| ; CHECK-NEXT: [[TMP6:%.*]] = zext <8 x i32> [[STEP_ADD_3]] to <8 x i64> |
| ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <8 x i64> [[BROADCAST_SPLAT]], [[TMP3]] |
| ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq <8 x i64> [[BROADCAST_SPLAT]], [[TMP4]] |
| ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq <8 x i64> [[BROADCAST_SPLAT]], [[TMP5]] |
| ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq <8 x i64> [[BROADCAST_SPLAT]], [[TMP6]] |
| ; CHECK-NEXT: [[TMP11:%.*]] = select <8 x i1> [[TMP7]], <8 x i32> zeroinitializer, <8 x i32> splat (i32 2) |
| ; CHECK-NEXT: [[TMP12:%.*]] = select <8 x i1> [[TMP8]], <8 x i32> zeroinitializer, <8 x i32> splat (i32 2) |
| ; CHECK-NEXT: [[TMP13:%.*]] = select <8 x i1> [[TMP9]], <8 x i32> zeroinitializer, <8 x i32> splat (i32 2) |
| ; CHECK-NEXT: [[TMP14:%.*]] = select <8 x i1> [[TMP10]], <8 x i32> zeroinitializer, <8 x i32> splat (i32 2) |
| ; CHECK-NEXT: [[TMP15]] = or <8 x i32> [[TMP11]], [[VEC_PHI]] |
| ; CHECK-NEXT: [[TMP16]] = or <8 x i32> [[TMP12]], [[VEC_PHI2]] |
| ; CHECK-NEXT: [[TMP17]] = or <8 x i32> [[TMP13]], [[VEC_PHI3]] |
| ; CHECK-NEXT: [[TMP18]] = or <8 x i32> [[TMP14]], [[VEC_PHI4]] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32 |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[STEP_ADD_3]], splat (i32 8) |
| ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[BIN_RDX:%.*]] = or <8 x i32> [[TMP16]], [[TMP15]] |
| ; CHECK-NEXT: [[BIN_RDX5:%.*]] = or <8 x i32> [[TMP17]], [[BIN_RDX]] |
| ; CHECK-NEXT: [[BIN_RDX6:%.*]] = or <8 x i32> [[TMP18]], [[BIN_RDX5]] |
| ; CHECK-NEXT: [[TMP20:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[BIN_RDX6]]) |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] |
| ; CHECK: [[VEC_EPILOG_ITER_CHECK]]: |
| ; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[N_MOD_VF]], 4 |
| ; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF24:![0-9]+]] |
| ; CHECK: [[VEC_EPILOG_PH]]: |
| ; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP20]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] |
| ; CHECK-NEXT: [[N_MOD_VF7:%.*]] = and i32 [[TMP1]], 3 |
| ; CHECK-NEXT: [[N_VEC8:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF7]] |
| ; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT10:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT9]], <4 x i64> poison, <4 x i32> zeroinitializer |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <4 x i32> poison, i32 [[VEC_EPILOG_RESUME_VAL]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT11]], <4 x i32> poison, <4 x i32> zeroinitializer |
| ; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT12]], <i32 0, i32 1, i32 2, i32 3> |
| ; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] |
| ; CHECK: [[VEC_EPILOG_VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX13:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_IND14:%.*]] = phi <4 x i32> [ [[INDUCTION]], %[[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT17:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[VEC_PHI15:%.*]] = phi <4 x i32> [ [[TMP21]], %[[VEC_EPILOG_PH]] ], [ [[TMP25:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[TMP22:%.*]] = zext <4 x i32> [[VEC_IND14]] to <4 x i64> |
| ; CHECK-NEXT: [[TMP23:%.*]] = icmp eq <4 x i64> [[BROADCAST_SPLAT10]], [[TMP22]] |
| ; CHECK-NEXT: [[TMP24:%.*]] = select <4 x i1> [[TMP23]], <4 x i32> zeroinitializer, <4 x i32> splat (i32 2) |
| ; CHECK-NEXT: [[TMP25]] = or <4 x i32> [[TMP24]], [[VEC_PHI15]] |
| ; CHECK-NEXT: [[INDEX_NEXT16]] = add nuw i32 [[INDEX13]], 4 |
| ; CHECK-NEXT: [[VEC_IND_NEXT17]] = add <4 x i32> [[VEC_IND14]], splat (i32 4) |
| ; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT16]], [[N_VEC8]] |
| ; CHECK-NEXT: br i1 [[TMP26]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] |
| ; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[TMP27:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP25]]) |
| ; CHECK-NEXT: [[CMP_N18:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC8]] |
| ; CHECK-NEXT: br i1 [[CMP_N18]], [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]] |
| ; CHECK: [[VEC_EPILOG_SCALAR_PH]]: |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] |
| %select = phi i32 [ 0, %entry ], [ %select.next, %loop ] |
| %iv.widen = zext i32 %iv to i64 |
| %exitcond = icmp eq i64 %n, %iv.widen |
| %select.i = select i1 %exitcond, i32 0, i32 2 |
| %select.next = or i32 %select.i, %select |
| %iv.next = add i32 %iv, 1 |
| br i1 %exitcond, label %exit, label %loop |
| |
| exit: |
| ret i32 %select.next |
| } |
| |
| ; Test case for https://github.com/llvm/llvm-project/issues/176720. |
| define void @replicating_sdiv_operand_profitable_to_scalarize(i32 %x) #3 { |
| ; CHECK-LABEL: define void @replicating_sdiv_operand_profitable_to_scalarize( |
| ; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR4:[0-9]+]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: br label %[[LOOP:.*]] |
| ; CHECK: [[LOOP]]: |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[X]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer |
| ; CHECK-NEXT: br label %[[EXIT:.*]] |
| ; CHECK: [[EXIT]]: |
| ; CHECK-NEXT: [[TMP0:%.*]] = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> splat (i32 2), <16 x i32> [[BROADCAST_SPLAT]], <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>) |
| ; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> zeroinitializer, <16 x i32> [[TMP0]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30> |
| ; CHECK-NEXT: [[TMP2:%.*]] = sub <16 x i32> [[BROADCAST_SPLAT]], [[TMP0]] |
| ; CHECK-NEXT: [[TMP3:%.*]] = or <16 x i32> [[TMP2]], [[TMP1]] |
| ; CHECK-NEXT: [[TMP4:%.*]] = or <16 x i32> [[TMP0]], splat (i32 3) |
| ; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> [[TMP0]], <16 x i32> [[TMP4]], <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>) |
| ; CHECK-NEXT: [[TMP6:%.*]] = or <16 x i32> [[TMP3]], [[TMP5]] |
| ; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> <i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true>, i1 false) |
| ; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1 |
| ; CHECK-NEXT: [[TMP7:%.*]] = extractelement <16 x i32> [[TMP6]], i64 [[LAST_ACTIVE_LANE]] |
| ; CHECK-NEXT: br label %[[EXIT1:.*]] |
| ; CHECK: [[EXIT1]]: |
| ; CHECK-NEXT: ret void |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i32 [ 0, %entry ], [ %div1, %loop ] |
| %accum = phi i32 [ 0, %entry ], [ %next, %loop ] |
| %div1 = sdiv i32 2, %x |
| %sub = sub i32 %x, %div1 |
| %or1 = or i32 %sub, %iv |
| %or2 = or i32 %div1, 3 |
| %div2 = sdiv i32 %div1, %or2 |
| %or3 = or i32 %or1, %div2 |
| %next = add i32 %accum, 3 |
| %cond = icmp ugt i32 %accum, 35 |
| br i1 %cond, label %exit, label %loop |
| |
| exit: |
| %result = phi i32 [ %or3, %loop ] |
| ret void |
| } |
| |
| @src.arr = external global [15 x double] |
| @dst.arr.a = external global [10 x double] |
| @dst.arr.b = external global [15 x double] |
| |
| define void @known_deref_load_tail_folding() #4 { |
| ; CHECK-LABEL: define void @known_deref_load_tail_folding( |
| ; CHECK-SAME: ) #[[ATTR5:[0-9]+]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ] |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ] |
| ; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 10) |
| ; CHECK-NEXT: [[TMP0:%.*]] = add i64 1, [[INDEX]] |
| ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr @src.arr, i64 [[TMP0]] |
| ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP2]], <4 x i1> [[TMP1]], <4 x double> poison) |
| ; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 |
| ; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] |
| ; CHECK: [[PRED_STORE_IF]]: |
| ; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[WIDE_MASKED_LOAD]], i64 0 |
| ; CHECK-NEXT: [[TMP5:%.*]] = fcmp oeq double [[TMP4]], 0.000000e+00 |
| ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], ptr @dst.arr.a, ptr @dst.arr.b |
| ; CHECK-NEXT: store double 0.000000e+00, ptr [[TMP6]], align 8 |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]] |
| ; CHECK: [[PRED_STORE_CONTINUE]]: |
| ; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 |
| ; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]] |
| ; CHECK: [[PRED_STORE_IF1]]: |
| ; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x double> [[WIDE_MASKED_LOAD]], i64 1 |
| ; CHECK-NEXT: [[TMP9:%.*]] = fcmp oeq double [[TMP8]], 0.000000e+00 |
| ; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], ptr @dst.arr.a, ptr @dst.arr.b |
| ; CHECK-NEXT: store double 0.000000e+00, ptr [[TMP10]], align 8 |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]] |
| ; CHECK: [[PRED_STORE_CONTINUE2]]: |
| ; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 |
| ; CHECK-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]] |
| ; CHECK: [[PRED_STORE_IF3]]: |
| ; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x double> [[WIDE_MASKED_LOAD]], i64 2 |
| ; CHECK-NEXT: [[TMP13:%.*]] = fcmp oeq double [[TMP12]], 0.000000e+00 |
| ; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], ptr @dst.arr.a, ptr @dst.arr.b |
| ; CHECK-NEXT: store double 0.000000e+00, ptr [[TMP14]], align 8 |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]] |
| ; CHECK: [[PRED_STORE_CONTINUE4]]: |
| ; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 |
| ; CHECK-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]] |
| ; CHECK: [[PRED_STORE_IF5]]: |
| ; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x double> [[WIDE_MASKED_LOAD]], i64 3 |
| ; CHECK-NEXT: [[TMP17:%.*]] = fcmp oeq double [[TMP16]], 0.000000e+00 |
| ; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP17]], ptr @dst.arr.a, ptr @dst.arr.b |
| ; CHECK-NEXT: store double 0.000000e+00, ptr [[TMP18]], align 8 |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]] |
| ; CHECK: [[PRED_STORE_CONTINUE6]]: |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i8> [[VEC_IND]], splat (i8 4) |
| ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 12 |
| ; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: br label %[[EXIT:.*]] |
| ; CHECK: [[EXIT]]: |
| ; CHECK-NEXT: ret void |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ] |
| %gep.src = getelementptr double, ptr @src.arr, i64 %iv |
| %val = load double, ptr %gep.src, align 8 |
| %cmp = fcmp oeq double %val, 0.000000e+00 |
| %dst.select = select i1 %cmp, ptr @dst.arr.a, ptr @dst.arr.b |
| store double 0.000000e+00, ptr %dst.select, align 8 |
| %iv.next = add i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, 12 |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
| |
| |
| attributes #0 = { "target-cpu"="penryn" } |
| attributes #1 = { "target-features"="+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl" } |
| attributes #2 = { "target-cpu"="znver3" } |
| attributes #3 = { "target-cpu"="skylake-avx512" } |
| attributes #4 = { "target-cpu"="haswell" } |
| |
| !0 = distinct !{!0, !1, !2, !3} |
| !1 = !{!"llvm.loop.vectorize.width", i32 2} |
| !2 = !{!"llvm.loop.vectorize.scalable.disable"} |
| !3 = !{!"llvm.loop.vectorize.enable"} |
