| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5 | |
| ; RUN: opt -p loop-vectorize -S %s | FileCheck --check-prefixes=COMMON,DEFAULT %s | |
| ; RUN: opt -p loop-vectorize -tail-folding-policy=prefer-fold-tail -S %s | FileCheck --check-prefixes=COMMON,PRED %s | |
| target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" | |
| target triple = "arm64-apple-macosx14.0.0" | |
| define void @invar_cond_gep_store(ptr %dst, i32 %0) { | |
| ; COMMON-LABEL: define void @invar_cond_gep_store( | |
| ; COMMON-SAME: ptr [[DST:%.*]], i32 [[TMP0:%.*]]) { | |
| ; COMMON-NEXT: [[ENTRY:.*]]: | |
| ; COMMON-NEXT: br label %[[LOOP_HEADER:.*]] | |
| ; COMMON: [[LOOP_HEADER]]: | |
| ; COMMON-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] | |
| ; COMMON-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 | |
| ; COMMON-NEXT: [[CMP9:%.*]] = icmp eq i32 [[TMP0]], 0 | |
| ; COMMON-NEXT: br i1 [[CMP9]], label %[[THEN:.*]], label %[[LOOP_LATCH]] | |
| ; COMMON: [[THEN]]: | |
| ; COMMON-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV_NEXT]] | |
| ; COMMON-NEXT: store i32 1, ptr [[GEP]], align 4 | |
| ; COMMON-NEXT: br label %[[LOOP_LATCH]] | |
| ; COMMON: [[LOOP_LATCH]]: | |
| ; COMMON-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 100 | |
| ; COMMON-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]] | |
| ; COMMON: [[EXIT]]: | |
| ; COMMON-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop.header | |
| loop.header: | |
| %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop.latch ] | |
| %iv.next = add i64 %iv, 1 | |
| %cmp9 = icmp eq i32 %0, 0 | |
| br i1 %cmp9, label %then, label %loop.latch | |
| then: | |
| %gep = getelementptr i32, ptr %dst, i64 %iv.next | |
| store i32 1, ptr %gep, align 4 | |
| br label %loop.latch | |
| loop.latch: | |
| %ec = icmp eq i64 %iv, 100 | |
| br i1 %ec, label %exit, label %loop.header | |
| exit: | |
| ret void | |
| } | |
| define void @loop_dependent_cond(ptr %src, ptr noalias %dst, i64 %N) { | |
| ; DEFAULT-LABEL: define void @loop_dependent_cond( | |
| ; DEFAULT-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) { | |
| ; DEFAULT-NEXT: [[ENTRY:.*:]] | |
| ; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 | |
| ; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4 | |
| ; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; DEFAULT: [[VECTOR_PH]]: | |
| ; DEFAULT-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 3 | |
| ; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]] | |
| ; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; DEFAULT: [[VECTOR_BODY]]: | |
| ; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ] | |
| ; DEFAULT-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]] | |
| ; DEFAULT-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i64 16 | |
| ; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP2]], align 8 | |
| ; DEFAULT-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP3]], align 8 | |
| ; DEFAULT-NEXT: [[TMP4:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[WIDE_LOAD]]) | |
| ; DEFAULT-NEXT: [[TMP5:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[WIDE_LOAD1]]) | |
| ; DEFAULT-NEXT: [[TMP6:%.*]] = fcmp ogt <2 x double> [[TMP4]], splat (double 1.000000e+00) | |
| ; DEFAULT-NEXT: [[TMP7:%.*]] = fcmp ogt <2 x double> [[TMP5]], splat (double 1.000000e+00) | |
| ; DEFAULT-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP6]], i64 0 | |
| ; DEFAULT-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] | |
| ; DEFAULT: [[PRED_STORE_IF]]: | |
| ; DEFAULT-NEXT: store i32 0, ptr [[DST]], align 4 | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE]]: | |
| ; DEFAULT-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP6]], i64 1 | |
| ; DEFAULT-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]] | |
| ; DEFAULT: [[PRED_STORE_IF2]]: | |
| ; DEFAULT-NEXT: store i32 0, ptr [[DST]], align 4 | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE3]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE3]]: | |
| ; DEFAULT-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP7]], i64 0 | |
| ; DEFAULT-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]] | |
| ; DEFAULT: [[PRED_STORE_IF4]]: | |
| ; DEFAULT-NEXT: store i32 0, ptr [[DST]], align 4 | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE5]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE5]]: | |
| ; DEFAULT-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP7]], i64 1 | |
| ; DEFAULT-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7]] | |
| ; DEFAULT: [[PRED_STORE_IF6]]: | |
| ; DEFAULT-NEXT: store i32 0, ptr [[DST]], align 4 | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE7]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE7]]: | |
| ; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; DEFAULT-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; DEFAULT-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] | |
| ; DEFAULT: [[MIDDLE_BLOCK]]: | |
| ; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] | |
| ; DEFAULT-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] | |
| ; DEFAULT: [[SCALAR_PH]]: | |
| ; | |
| ; PRED-LABEL: define void @loop_dependent_cond( | |
| ; PRED-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) { | |
| ; PRED-NEXT: [[ENTRY:.*]]: | |
| ; PRED-NEXT: br label %[[LOOP_HEADER:.*]] | |
| ; PRED: [[LOOP_HEADER]]: | |
| ; PRED-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] | |
| ; PRED-NEXT: [[GEP:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV]] | |
| ; PRED-NEXT: [[L:%.*]] = load double, ptr [[GEP]], align 8 | |
| ; PRED-NEXT: [[ABS:%.*]] = tail call double @llvm.fabs.f64(double [[L]]) | |
| ; PRED-NEXT: [[CMP:%.*]] = fcmp ogt double [[ABS]], 1.000000e+00 | |
| ; PRED-NEXT: br i1 [[CMP]], label %[[THEN:.*]], label %[[LOOP_LATCH]] | |
| ; PRED: [[THEN]]: | |
| ; PRED-NEXT: store i32 0, ptr [[DST]], align 4 | |
| ; PRED-NEXT: br label %[[LOOP_LATCH]] | |
| ; PRED: [[LOOP_LATCH]]: | |
| ; PRED-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 | |
| ; PRED-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]] | |
| ; PRED-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]] | |
| ; PRED: [[EXIT]]: | |
| ; PRED-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop.header | |
| loop.header: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] | |
| %gep = getelementptr double, ptr %src, i64 %iv | |
| %l = load double, ptr %gep, align 8 | |
| %abs = tail call double @llvm.fabs.f64(double %l) | |
| %cmp = fcmp ogt double %abs, 1.000000e+00 | |
| br i1 %cmp, label %then, label %loop.latch | |
| then: | |
| store i32 0, ptr %dst, align 4 | |
| br label %loop.latch | |
| loop.latch: | |
| %iv.next = add i64 %iv, 1 | |
| %ec = icmp eq i64 %iv, %N | |
| br i1 %ec, label %exit, label %loop.header | |
| exit: | |
| ret void | |
| } | |
| define void @invar_cond_chain_1(ptr %I, ptr noalias %src, i1 %c) { | |
| ; COMMON-LABEL: define void @invar_cond_chain_1( | |
| ; COMMON-SAME: ptr [[I:%.*]], ptr noalias [[SRC:%.*]], i1 [[C:%.*]]) { | |
| ; COMMON-NEXT: [[ENTRY:.*]]: | |
| ; COMMON-NEXT: br label %[[LOOP_HEADER:.*]] | |
| ; COMMON: [[LOOP_HEADER]]: | |
| ; COMMON-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] | |
| ; COMMON-NEXT: [[L:%.*]] = load i32, ptr [[PTR_IV]], align 4 | |
| ; COMMON-NEXT: br i1 true, label %[[ELSE_1:.*]], label %[[IF:.*]] | |
| ; COMMON: [[IF]]: | |
| ; COMMON-NEXT: br label %[[SPLIT:.*]] | |
| ; COMMON: [[ELSE_1]]: | |
| ; COMMON-NEXT: br i1 [[C]], label %[[ELSE_2:.*]], label %[[LOOP_LATCH]] | |
| ; COMMON: [[ELSE_2]]: | |
| ; COMMON-NEXT: br label %[[SPLIT]] | |
| ; COMMON: [[SPLIT]]: | |
| ; COMMON-NEXT: store i32 [[L]], ptr [[I]], align 4 | |
| ; COMMON-NEXT: br label %[[LOOP_LATCH]] | |
| ; COMMON: [[LOOP_LATCH]]: | |
| ; COMMON-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV]], i64 4 | |
| ; COMMON-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IV]], [[I]] | |
| ; COMMON-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]] | |
| ; COMMON: [[EXIT]]: | |
| ; COMMON-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop.header | |
| loop.header: | |
| %ptr.iv = phi ptr [ %src, %entry ], [ %ptr.iv.next, %loop.latch ] | |
| %l = load i32, ptr %ptr.iv, align 4 | |
| br i1 true, label %else.1, label %if | |
| if: | |
| br label %split | |
| else.1: | |
| br i1 %c, label %else.2, label %loop.latch | |
| else.2: | |
| br label %split | |
| split: | |
| store i32 %l, ptr %I, align 4 | |
| br label %loop.latch | |
| loop.latch: | |
| %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 4 | |
| %ec = icmp eq ptr %ptr.iv, %I | |
| br i1 %ec, label %exit, label %loop.header | |
| exit: | |
| ret void | |
| } | |
| define void @invar_cond_chain_2(ptr %I, ptr noalias %src, ptr noalias %dst, i32 %a) { | |
| ; COMMON-LABEL: define void @invar_cond_chain_2( | |
| ; COMMON-SAME: ptr [[I:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i32 [[A:%.*]]) { | |
| ; COMMON-NEXT: [[ENTRY:.*]]: | |
| ; COMMON-NEXT: br label %[[LOOP_HEADER:.*]] | |
| ; COMMON: [[LOOP_HEADER]]: | |
| ; COMMON-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] | |
| ; COMMON-NEXT: [[CMP315_NOT:%.*]] = icmp sgt i32 [[A]], 0 | |
| ; COMMON-NEXT: br i1 [[CMP315_NOT]], label %[[LOOP_LATCH]], label %[[IF:.*]] | |
| ; COMMON: [[IF]]: | |
| ; COMMON-NEXT: br label %[[ELSE:.*]] | |
| ; COMMON: [[ELSE]]: | |
| ; COMMON-NEXT: store i32 0, ptr [[DST]], align 4 | |
| ; COMMON-NEXT: br label %[[LOOP_LATCH]] | |
| ; COMMON: [[LOOP_LATCH]]: | |
| ; COMMON-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV]], i64 4 | |
| ; COMMON-NEXT: [[CMP311_NOT:%.*]] = icmp eq ptr [[PTR_IV]], [[I]] | |
| ; COMMON-NEXT: br i1 [[CMP311_NOT]], label %[[EXIT:.*]], label %[[LOOP_HEADER]] | |
| ; COMMON: [[EXIT]]: | |
| ; COMMON-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop.header | |
| loop.header: | |
| %ptr.iv = phi ptr [ %src, %entry ], [ %ptr.iv.next, %loop.latch ] | |
| %cmp315.not = icmp sgt i32 %a, 0 | |
| br i1 %cmp315.not, label %loop.latch, label %if | |
| if: | |
| br label %else | |
| else: | |
| store i32 0, ptr %dst, align 4 | |
| br label %loop.latch | |
| loop.latch: | |
| %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 4 | |
| %cmp311.not = icmp eq ptr %ptr.iv, %I | |
| br i1 %cmp311.not, label %exit, label %loop.header | |
| exit: | |
| ret void | |
| } | |
| define void @latch_branch_cost(ptr %dst) { | |
| ; DEFAULT-LABEL: define void @latch_branch_cost( | |
| ; DEFAULT-SAME: ptr [[DST:%.*]]) { | |
| ; DEFAULT-NEXT: [[ITER_CHECK:.*:]] | |
| ; DEFAULT-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] | |
| ; DEFAULT: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: | |
| ; DEFAULT-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; DEFAULT: [[VECTOR_PH]]: | |
| ; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; DEFAULT: [[VECTOR_BODY]]: | |
| ; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; DEFAULT-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]] | |
| ; DEFAULT-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i64 16 | |
| ; DEFAULT-NEXT: store <16 x i8> zeroinitializer, ptr [[TMP0]], align 1 | |
| ; DEFAULT-NEXT: store <16 x i8> zeroinitializer, ptr [[TMP1]], align 1 | |
| ; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32 | |
| ; DEFAULT-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96 | |
| ; DEFAULT-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] | |
| ; DEFAULT: [[MIDDLE_BLOCK]]: | |
| ; DEFAULT-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] | |
| ; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]: | |
| ; DEFAULT-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]] | |
| ; DEFAULT: [[VEC_EPILOG_PH]]: | |
| ; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] | |
| ; DEFAULT-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] | |
| ; DEFAULT: [[VEC_EPILOG_VECTOR_BODY]]: | |
| ; DEFAULT-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] | |
| ; DEFAULT-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX1]] | |
| ; DEFAULT-NEXT: store <4 x i8> zeroinitializer, ptr [[TMP3]], align 1 | |
| ; DEFAULT-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4 | |
| ; DEFAULT-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 100 | |
| ; DEFAULT-NEXT: br i1 [[TMP4]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] | |
| ; DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]: | |
| ; DEFAULT-NEXT: br i1 true, [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]] | |
| ; DEFAULT: [[VEC_EPILOG_SCALAR_PH]]: | |
| ; | |
| ; PRED-LABEL: define void @latch_branch_cost( | |
| ; PRED-SAME: ptr [[DST:%.*]]) { | |
| ; PRED-NEXT: [[ENTRY:.*]]: | |
| ; PRED-NEXT: br label %[[LOOP:.*]] | |
| ; PRED: [[LOOP]]: | |
| ; PRED-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; PRED-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV]] | |
| ; PRED-NEXT: store i8 0, ptr [[GEP]], align 1 | |
| ; PRED-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 | |
| ; PRED-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100 | |
| ; PRED-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]] | |
| ; PRED: [[EXIT]]: | |
| ; PRED-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %gep = getelementptr i8, ptr %dst, i64 %iv | |
| store i8 0, ptr %gep, align 1 | |
| %iv.next = add i64 %iv, 1 | |
| %ec = icmp eq i64 %iv.next, 100 | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, ptr %E, i64 %N) "target-features"="+sve" vscale_range(1,16) { | |
| ; DEFAULT-LABEL: define i32 @header_mask_and_invariant_compare( | |
| ; DEFAULT-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]], ptr [[E:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { | |
| ; DEFAULT-NEXT: [[ENTRY:.*:]] | |
| ; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 | |
| ; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 28 | |
| ; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] | |
| ; DEFAULT: [[VECTOR_MEMCHECK]]: | |
| ; DEFAULT-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[E]], i64 4 | |
| ; DEFAULT-NEXT: [[TMP2:%.*]] = shl i64 [[N]], 2 | |
| ; DEFAULT-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], 4 | |
| ; DEFAULT-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[D]], i64 [[TMP3]] | |
| ; DEFAULT-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 4 | |
| ; DEFAULT-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 4 | |
| ; DEFAULT-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[C]], i64 4 | |
| ; DEFAULT-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[E]], [[TMP4]] | |
| ; DEFAULT-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[D]], [[TMP1]] | |
| ; DEFAULT-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] | |
| ; DEFAULT-NEXT: [[BOUND028:%.*]] = icmp ult ptr [[E]], [[TMP5]] | |
| ; DEFAULT-NEXT: [[BOUND129:%.*]] = icmp ult ptr [[A]], [[TMP1]] | |
| ; DEFAULT-NEXT: [[FOUND_CONFLICT30:%.*]] = and i1 [[BOUND028]], [[BOUND129]] | |
| ; DEFAULT-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT30]] | |
| ; DEFAULT-NEXT: [[BOUND031:%.*]] = icmp ult ptr [[E]], [[TMP6]] | |
| ; DEFAULT-NEXT: [[BOUND132:%.*]] = icmp ult ptr [[B]], [[TMP1]] | |
| ; DEFAULT-NEXT: [[FOUND_CONFLICT33:%.*]] = and i1 [[BOUND031]], [[BOUND132]] | |
| ; DEFAULT-NEXT: [[CONFLICT_RDX34:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT33]] | |
| ; DEFAULT-NEXT: [[BOUND035:%.*]] = icmp ult ptr [[E]], [[TMP7]] | |
| ; DEFAULT-NEXT: [[BOUND136:%.*]] = icmp ult ptr [[C]], [[TMP1]] | |
| ; DEFAULT-NEXT: [[FOUND_CONFLICT37:%.*]] = and i1 [[BOUND035]], [[BOUND136]] | |
| ; DEFAULT-NEXT: [[CONFLICT_RDX38:%.*]] = or i1 [[CONFLICT_RDX34]], [[FOUND_CONFLICT37]] | |
| ; DEFAULT-NEXT: [[BOUND039:%.*]] = icmp ult ptr [[D]], [[TMP5]] | |
| ; DEFAULT-NEXT: [[BOUND140:%.*]] = icmp ult ptr [[A]], [[TMP4]] | |
| ; DEFAULT-NEXT: [[FOUND_CONFLICT41:%.*]] = and i1 [[BOUND039]], [[BOUND140]] | |
| ; DEFAULT-NEXT: [[CONFLICT_RDX42:%.*]] = or i1 [[CONFLICT_RDX38]], [[FOUND_CONFLICT41]] | |
| ; DEFAULT-NEXT: [[BOUND043:%.*]] = icmp ult ptr [[D]], [[TMP6]] | |
| ; DEFAULT-NEXT: [[BOUND144:%.*]] = icmp ult ptr [[B]], [[TMP4]] | |
| ; DEFAULT-NEXT: [[FOUND_CONFLICT45:%.*]] = and i1 [[BOUND043]], [[BOUND144]] | |
| ; DEFAULT-NEXT: [[CONFLICT_RDX46:%.*]] = or i1 [[CONFLICT_RDX42]], [[FOUND_CONFLICT45]] | |
| ; DEFAULT-NEXT: [[BOUND047:%.*]] = icmp ult ptr [[D]], [[TMP7]] | |
| ; DEFAULT-NEXT: [[BOUND148:%.*]] = icmp ult ptr [[C]], [[TMP4]] | |
| ; DEFAULT-NEXT: [[FOUND_CONFLICT49:%.*]] = and i1 [[BOUND047]], [[BOUND148]] | |
| ; DEFAULT-NEXT: [[CONFLICT_RDX50:%.*]] = or i1 [[CONFLICT_RDX46]], [[FOUND_CONFLICT49]] | |
| ; DEFAULT-NEXT: br i1 [[CONFLICT_RDX50]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] | |
| ; DEFAULT: [[VECTOR_PH]]: | |
| ; DEFAULT-NEXT: [[TMP8:%.*]] = and i64 [[TMP0]], 3 | |
| ; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP8]] | |
| ; DEFAULT-NEXT: [[TMP9:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META8:![0-9]+]] | |
| ; DEFAULT-NEXT: [[TMP10:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META11:![0-9]+]] | |
| ; DEFAULT-NEXT: [[TMP11:%.*]] = or i32 [[TMP10]], [[TMP9]] | |
| ; DEFAULT-NEXT: [[TMP12:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META13:![0-9]+]] | |
| ; DEFAULT-NEXT: [[TMP13:%.*]] = icmp ugt i32 [[TMP12]], [[TMP11]] | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP13]], i64 0 | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer | |
| ; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; DEFAULT: [[VECTOR_BODY]]: | |
| ; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE56:.*]] ] | |
| ; DEFAULT-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[D]], i64 [[INDEX]] | |
| ; DEFAULT-NEXT: br i1 [[TMP13]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] | |
| ; DEFAULT: [[PRED_STORE_IF]]: | |
| ; DEFAULT-NEXT: store i32 [[TMP11]], ptr [[E]], align 4, !alias.scope [[META15:![0-9]+]], !noalias [[META17:![0-9]+]] | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE]]: | |
| ; DEFAULT-NEXT: br i1 [[TMP13]], label %[[PRED_STORE_IF51:.*]], label %[[PRED_STORE_CONTINUE52:.*]] | |
| ; DEFAULT: [[PRED_STORE_IF51]]: | |
| ; DEFAULT-NEXT: store i32 [[TMP11]], ptr [[E]], align 4, !alias.scope [[META15]], !noalias [[META17]] | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE52]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE52]]: | |
| ; DEFAULT-NEXT: br i1 [[TMP13]], label %[[PRED_STORE_IF53:.*]], label %[[PRED_STORE_CONTINUE54:.*]] | |
| ; DEFAULT: [[PRED_STORE_IF53]]: | |
| ; DEFAULT-NEXT: store i32 [[TMP11]], ptr [[E]], align 4, !alias.scope [[META15]], !noalias [[META17]] | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE54]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE54]]: | |
| ; DEFAULT-NEXT: br i1 [[TMP13]], label %[[PRED_STORE_IF55:.*]], label %[[PRED_STORE_CONTINUE56]] | |
| ; DEFAULT: [[PRED_STORE_IF55]]: | |
| ; DEFAULT-NEXT: store i32 [[TMP11]], ptr [[E]], align 4, !alias.scope [[META15]], !noalias [[META17]] | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE56]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE56]]: | |
| ; DEFAULT-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> zeroinitializer, ptr align 4 [[TMP14]], <4 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META19:![0-9]+]], !noalias [[META20:![0-9]+]] | |
| ; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; DEFAULT-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; DEFAULT-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] | |
| ; DEFAULT: [[MIDDLE_BLOCK]]: | |
| ; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] | |
| ; DEFAULT-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] | |
| ; DEFAULT: [[SCALAR_PH]]: | |
| ; | |
| ; PRED-LABEL: define i32 @header_mask_and_invariant_compare( | |
| ; PRED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]], ptr [[E:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { | |
| ; PRED-NEXT: [[ENTRY:.*:]] | |
| ; PRED-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 | |
| ; PRED-NEXT: br label %[[VECTOR_MEMCHECK:.*]] | |
| ; PRED: [[VECTOR_MEMCHECK]]: | |
| ; PRED-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[E]], i64 4 | |
| ; PRED-NEXT: [[TMP2:%.*]] = shl i64 [[N]], 2 | |
| ; PRED-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], 4 | |
| ; PRED-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[D]], i64 [[TMP3]] | |
| ; PRED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 4 | |
| ; PRED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 4 | |
| ; PRED-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[C]], i64 4 | |
| ; PRED-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[E]], [[TMP4]] | |
| ; PRED-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[D]], [[TMP1]] | |
| ; PRED-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] | |
| ; PRED-NEXT: [[BOUND028:%.*]] = icmp ult ptr [[E]], [[TMP5]] | |
| ; PRED-NEXT: [[BOUND129:%.*]] = icmp ult ptr [[A]], [[TMP1]] | |
| ; PRED-NEXT: [[FOUND_CONFLICT30:%.*]] = and i1 [[BOUND028]], [[BOUND129]] | |
| ; PRED-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT30]] | |
| ; PRED-NEXT: [[BOUND031:%.*]] = icmp ult ptr [[E]], [[TMP6]] | |
| ; PRED-NEXT: [[BOUND132:%.*]] = icmp ult ptr [[B]], [[TMP1]] | |
| ; PRED-NEXT: [[FOUND_CONFLICT33:%.*]] = and i1 [[BOUND031]], [[BOUND132]] | |
| ; PRED-NEXT: [[CONFLICT_RDX34:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT33]] | |
| ; PRED-NEXT: [[BOUND035:%.*]] = icmp ult ptr [[E]], [[TMP7]] | |
| ; PRED-NEXT: [[BOUND136:%.*]] = icmp ult ptr [[C]], [[TMP1]] | |
| ; PRED-NEXT: [[FOUND_CONFLICT37:%.*]] = and i1 [[BOUND035]], [[BOUND136]] | |
| ; PRED-NEXT: [[CONFLICT_RDX38:%.*]] = or i1 [[CONFLICT_RDX34]], [[FOUND_CONFLICT37]] | |
| ; PRED-NEXT: [[BOUND039:%.*]] = icmp ult ptr [[D]], [[TMP5]] | |
| ; PRED-NEXT: [[BOUND140:%.*]] = icmp ult ptr [[A]], [[TMP4]] | |
| ; PRED-NEXT: [[FOUND_CONFLICT41:%.*]] = and i1 [[BOUND039]], [[BOUND140]] | |
| ; PRED-NEXT: [[CONFLICT_RDX42:%.*]] = or i1 [[CONFLICT_RDX38]], [[FOUND_CONFLICT41]] | |
| ; PRED-NEXT: [[BOUND043:%.*]] = icmp ult ptr [[D]], [[TMP6]] | |
| ; PRED-NEXT: [[BOUND144:%.*]] = icmp ult ptr [[B]], [[TMP4]] | |
| ; PRED-NEXT: [[FOUND_CONFLICT45:%.*]] = and i1 [[BOUND043]], [[BOUND144]] | |
| ; PRED-NEXT: [[CONFLICT_RDX46:%.*]] = or i1 [[CONFLICT_RDX42]], [[FOUND_CONFLICT45]] | |
| ; PRED-NEXT: [[BOUND047:%.*]] = icmp ult ptr [[D]], [[TMP7]] | |
| ; PRED-NEXT: [[BOUND148:%.*]] = icmp ult ptr [[C]], [[TMP4]] | |
| ; PRED-NEXT: [[FOUND_CONFLICT49:%.*]] = and i1 [[BOUND047]], [[BOUND148]] | |
| ; PRED-NEXT: [[CONFLICT_RDX50:%.*]] = or i1 [[CONFLICT_RDX46]], [[FOUND_CONFLICT49]] | |
| ; PRED-NEXT: br i1 [[CONFLICT_RDX50]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; PRED: [[VECTOR_PH]]: | |
| ; PRED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() | |
| ; PRED-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 2 | |
| ; PRED-NEXT: [[TMP10:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META0:![0-9]+]] | |
| ; PRED-NEXT: [[TMP11:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META3:![0-9]+]] | |
| ; PRED-NEXT: [[TMP12:%.*]] = or i32 [[TMP11]], [[TMP10]] | |
| ; PRED-NEXT: [[BROADCAST_SPLATINSERT53:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0 | |
| ; PRED-NEXT: [[BROADCAST_SPLAT54:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT53]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer | |
| ; PRED-NEXT: [[TMP13:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META5:![0-9]+]] | |
| ; PRED-NEXT: [[TMP14:%.*]] = icmp ugt i32 [[TMP13]], [[TMP12]] | |
| ; PRED-NEXT: [[BROADCAST_SPLATINSERT51:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[TMP14]], i64 0 | |
| ; PRED-NEXT: [[BROADCAST_SPLAT52:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT51]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer | |
| ; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]]) | |
| ; PRED-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[TMP0]], i64 [[TMP9]]) | |
| ; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[E]], i64 0 | |
| ; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer | |
| ; PRED-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; PRED: [[VECTOR_BODY]]: | |
| ; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; PRED-NEXT: [[TMP15:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[BROADCAST_SPLAT52]], <vscale x 4 x i1> zeroinitializer | |
| ; PRED-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[D]], i64 [[INDEX]] | |
| ; PRED-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT54]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP15]]), !alias.scope [[META7:![0-9]+]], !noalias [[META9:![0-9]+]] | |
| ; PRED-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> zeroinitializer, ptr align 4 [[TMP16]], <vscale x 4 x i1> [[TMP15]]), !alias.scope [[META11:![0-9]+]], !noalias [[META12:![0-9]+]] | |
| ; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TRIP_COUNT_MINUS_VF]]) | |
| ; PRED-NEXT: [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0 | |
| ; PRED-NEXT: [[TMP18:%.*]] = xor i1 [[TMP17]], true | |
| ; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] | |
| ; PRED-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] | |
| ; PRED: [[MIDDLE_BLOCK]]: | |
| ; PRED-NEXT: br [[EXIT:label %.*]] | |
| ; PRED: [[SCALAR_PH]]: | |
| ; | |
| entry: | |
| br label %loop.header | |
| loop.header: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] | |
| %l.A = load i32, ptr %A, align 4 | |
| %l.B = load i32, ptr %B, align 4 | |
| %or = or i32 %l.B, %l.A | |
| %l.C = load i32, ptr %C, align 4 | |
| %c.0 = icmp ugt i32 %l.C, %or | |
| br i1 %c.0, label %if.then, label %loop.latch | |
| if.then: | |
| %gep.D = getelementptr i32, ptr %D, i64 %iv | |
| store i32 %or, ptr %E, align 4 | |
| store i32 0, ptr %gep.D, align 4 | |
| br label %loop.latch | |
| loop.latch: | |
| %iv.next = add i64 %iv, 1 | |
| %c.1 = icmp eq i64 %iv, %N | |
| br i1 %c.1, label %exit, label %loop.header | |
| exit: | |
| ret i32 0 | |
| } | |
| define void @multiple_exit_conditions(ptr %src, ptr noalias %dst) #1 { | |
| ; DEFAULT-LABEL: define void @multiple_exit_conditions( | |
| ; DEFAULT-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR1:[0-9]+]] { | |
| ; DEFAULT-NEXT: [[ENTRY:.*:]] | |
| ; DEFAULT-NEXT: br label %[[VECTOR_PH:.*]] | |
| ; DEFAULT: [[VECTOR_PH]]: | |
| ; DEFAULT-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() | |
| ; DEFAULT-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2 | |
| ; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4 | |
| ; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 257, [[TMP2]] | |
| ; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 257, [[N_MOD_VF]] | |
| ; DEFAULT-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 3 | |
| ; DEFAULT-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]] | |
| ; DEFAULT-NEXT: [[TMP5:%.*]] = shl i64 [[N_VEC]], 1 | |
| ; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; DEFAULT: [[VECTOR_BODY]]: | |
| ; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; DEFAULT-NEXT: [[TMP6:%.*]] = shl i64 [[INDEX]], 3 | |
| ; DEFAULT-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]] | |
| ; DEFAULT-NEXT: [[TMP7:%.*]] = load i16, ptr [[SRC]], align 2 | |
| ; DEFAULT-NEXT: [[TMP8:%.*]] = or i16 [[TMP7]], 1 | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[TMP8]], i64 0 | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer | |
| ; DEFAULT-NEXT: [[TMP9:%.*]] = uitofp <vscale x 4 x i16> [[BROADCAST_SPLAT]] to <vscale x 4 x double> | |
| ; DEFAULT-NEXT: [[TMP10:%.*]] = shl nuw nsw i64 [[TMP1]], 1 | |
| ; DEFAULT-NEXT: [[TMP11:%.*]] = mul nuw nsw i64 [[TMP1]], 3 | |
| ; DEFAULT-NEXT: [[TMP12:%.*]] = getelementptr [8 x i8], ptr [[NEXT_GEP]], i64 [[TMP1]] | |
| ; DEFAULT-NEXT: [[TMP13:%.*]] = getelementptr [8 x i8], ptr [[NEXT_GEP]], i64 [[TMP10]] | |
| ; DEFAULT-NEXT: [[TMP14:%.*]] = getelementptr [8 x i8], ptr [[NEXT_GEP]], i64 [[TMP11]] | |
| ; DEFAULT-NEXT: store <vscale x 4 x double> [[TMP9]], ptr [[NEXT_GEP]], align 8 | |
| ; DEFAULT-NEXT: store <vscale x 4 x double> [[TMP9]], ptr [[TMP12]], align 8 | |
| ; DEFAULT-NEXT: store <vscale x 4 x double> [[TMP9]], ptr [[TMP13]], align 8 | |
| ; DEFAULT-NEXT: store <vscale x 4 x double> [[TMP9]], ptr [[TMP14]], align 8 | |
| ; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]] | |
| ; DEFAULT-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; DEFAULT-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] | |
| ; DEFAULT: [[MIDDLE_BLOCK]]: | |
| ; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 257, [[N_VEC]] | |
| ; DEFAULT-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH:.*]] | |
| ; DEFAULT: [[SCALAR_PH]]: | |
| ; | |
| ; PRED-LABEL: define void @multiple_exit_conditions( | |
| ; PRED-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR1:[0-9]+]] { | |
| ; PRED-NEXT: [[ENTRY:.*:]] | |
| ; PRED-NEXT: br label %[[VECTOR_PH:.*]] | |
| ; PRED: [[VECTOR_PH]]: | |
| ; PRED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() | |
| ; PRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 | |
| ; PRED-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; PRED: [[VECTOR_BODY]]: | |
| ; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ splat (i1 true), %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; PRED-NEXT: [[TMP2:%.*]] = shl i64 [[INDEX]], 3 | |
| ; PRED-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]] | |
| ; PRED-NEXT: [[TMP3:%.*]] = load i16, ptr [[SRC]], align 2 | |
| ; PRED-NEXT: [[TMP4:%.*]] = or i16 [[TMP3]], 1 | |
| ; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i16> poison, i16 [[TMP4]], i64 0 | |
| ; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer | |
| ; PRED-NEXT: [[TMP5:%.*]] = uitofp <vscale x 2 x i16> [[BROADCAST_SPLAT]] to <vscale x 2 x double> | |
| ; PRED-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP5]], ptr align 8 [[NEXT_GEP]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]]) | |
| ; PRED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]] | |
| ; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 257) | |
| ; PRED-NEXT: [[TMP6:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0 | |
| ; PRED-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true | |
| ; PRED-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] | |
| ; PRED: [[MIDDLE_BLOCK]]: | |
| ; PRED-NEXT: br label %[[EXIT:.*]] | |
| ; PRED: [[EXIT]]: | |
| ; PRED-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ] | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %l = load i16, ptr %src, align 2 | |
| %o = or i16 %l, 1 | |
| %conv = uitofp i16 %o to double | |
| store double %conv, ptr %ptr.iv, align 8 | |
| %iv.next = add nsw i64 %iv, 2 | |
| %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 8 | |
| %iv.clamp = and i64 %iv, 4294967294 | |
| %ec = icmp eq i64 %iv.clamp, 512 | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| define void @low_trip_count_fold_tail_scalarized_store(ptr %dst) { | |
| ; COMMON-LABEL: define void @low_trip_count_fold_tail_scalarized_store( | |
| ; COMMON-SAME: ptr [[DST:%.*]]) { | |
| ; COMMON-NEXT: [[ENTRY:.*:]] | |
| ; COMMON-NEXT: br label %[[VECTOR_PH:.*]] | |
| ; COMMON: [[VECTOR_PH]]: | |
| ; COMMON-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; COMMON: [[VECTOR_BODY]]: | |
| ; COMMON-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] | |
| ; COMMON: [[PRED_STORE_IF]]: | |
| ; COMMON-NEXT: store i8 0, ptr [[DST]], align 1 | |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE]] | |
| ; COMMON: [[PRED_STORE_CONTINUE]]: | |
| ; COMMON-NEXT: br i1 true, label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]] | |
| ; COMMON: [[PRED_STORE_IF1]]: | |
| ; COMMON-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[DST]], i64 1 | |
| ; COMMON-NEXT: store i8 1, ptr [[TMP0]], align 1 | |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE2]] | |
| ; COMMON: [[PRED_STORE_CONTINUE2]]: | |
| ; COMMON-NEXT: br i1 true, label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]] | |
| ; COMMON: [[PRED_STORE_IF3]]: | |
| ; COMMON-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 2 | |
| ; COMMON-NEXT: store i8 2, ptr [[TMP1]], align 1 | |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE4]] | |
| ; COMMON: [[PRED_STORE_CONTINUE4]]: | |
| ; COMMON-NEXT: br i1 true, label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]] | |
| ; COMMON: [[PRED_STORE_IF5]]: | |
| ; COMMON-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[DST]], i64 3 | |
| ; COMMON-NEXT: store i8 3, ptr [[TMP2]], align 1 | |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE6]] | |
| ; COMMON: [[PRED_STORE_CONTINUE6]]: | |
| ; COMMON-NEXT: br i1 true, label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]] | |
| ; COMMON: [[PRED_STORE_IF7]]: | |
| ; COMMON-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 4 | |
| ; COMMON-NEXT: store i8 4, ptr [[TMP3]], align 1 | |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE8]] | |
| ; COMMON: [[PRED_STORE_CONTINUE8]]: | |
| ; COMMON-NEXT: br i1 true, label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]] | |
| ; COMMON: [[PRED_STORE_IF9]]: | |
| ; COMMON-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 5 | |
| ; COMMON-NEXT: store i8 5, ptr [[TMP4]], align 1 | |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE10]] | |
| ; COMMON: [[PRED_STORE_CONTINUE10]]: | |
| ; COMMON-NEXT: br i1 true, label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]] | |
| ; COMMON: [[PRED_STORE_IF11]]: | |
| ; COMMON-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[DST]], i64 6 | |
| ; COMMON-NEXT: store i8 6, ptr [[TMP5]], align 1 | |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE12]] | |
| ; COMMON: [[PRED_STORE_CONTINUE12]]: | |
| ; COMMON-NEXT: br i1 false, label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]] | |
| ; COMMON: [[PRED_STORE_IF13]]: | |
| ; COMMON-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 7 | |
| ; COMMON-NEXT: store i8 7, ptr [[TMP6]], align 1 | |
| ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE14]] | |
| ; COMMON: [[PRED_STORE_CONTINUE14]]: | |
| ; COMMON-NEXT: br label %[[MIDDLE_BLOCK:.*]] | |
| ; COMMON: [[MIDDLE_BLOCK]]: | |
| ; COMMON-NEXT: br label %[[EXIT:.*]] | |
| ; COMMON: [[EXIT]]: | |
| ; COMMON-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %iv.trunc = trunc i64 %iv to i8 | |
| %gep = getelementptr i8, ptr %dst, i64 %iv | |
| store i8 %iv.trunc, ptr %gep, align 1 | |
| %iv.next = add i64 %iv, 1 | |
| %ec = icmp eq i64 %iv.next, 7 | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias %src.2, ptr noalias %src.3, ptr noalias %src.4, ptr noalias %dst, i64 %N) #2 { | |
| ; DEFAULT-LABEL: define void @test_conditional_interleave_group( | |
| ; DEFAULT-SAME: ptr noalias [[SRC_1:%.*]], ptr noalias [[SRC_2:%.*]], ptr noalias [[SRC_3:%.*]], ptr noalias [[SRC_4:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) #[[ATTR2:[0-9]+]] { | |
| ; DEFAULT-NEXT: [[ENTRY:.*:]] | |
| ; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 | |
| ; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8 | |
| ; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]] | |
| ; DEFAULT: [[VECTOR_SCEVCHECK]]: | |
| ; DEFAULT-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]]) | |
| ; DEFAULT-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0 | |
| ; DEFAULT-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1 | |
| ; DEFAULT-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]] | |
| ; DEFAULT-NEXT: [[TMP2:%.*]] = icmp ult ptr [[TMP1]], [[DST]] | |
| ; DEFAULT-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]] | |
| ; DEFAULT-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4 | |
| ; DEFAULT-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]] | |
| ; DEFAULT-NEXT: [[TMP5:%.*]] = icmp ult ptr [[TMP4]], [[SCEVGEP]] | |
| ; DEFAULT-NEXT: [[TMP6:%.*]] = or i1 [[TMP5]], [[MUL_OVERFLOW]] | |
| ; DEFAULT-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 8 | |
| ; DEFAULT-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[MUL_RESULT]] | |
| ; DEFAULT-NEXT: [[TMP8:%.*]] = icmp ult ptr [[TMP7]], [[SCEVGEP1]] | |
| ; DEFAULT-NEXT: [[TMP9:%.*]] = or i1 [[TMP8]], [[MUL_OVERFLOW]] | |
| ; DEFAULT-NEXT: [[TMP10:%.*]] = or i1 [[TMP3]], [[TMP6]] | |
| ; DEFAULT-NEXT: [[TMP11:%.*]] = or i1 [[TMP10]], [[TMP9]] | |
| ; DEFAULT-NEXT: br i1 [[TMP11]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] | |
| ; DEFAULT: [[VECTOR_PH]]: | |
| ; DEFAULT-NEXT: [[TMP12:%.*]] = and i64 [[TMP0]], 7 | |
| ; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP12]] | |
| ; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; DEFAULT: [[VECTOR_BODY]]: | |
| ; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE19:.*]] ] | |
| ; DEFAULT-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE19]] ] | |
| ; DEFAULT-NEXT: [[TMP13:%.*]] = load float, ptr [[SRC_1]], align 4 | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[TMP13]], i64 0 | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x float> [[BROADCAST_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer | |
| ; DEFAULT-NEXT: [[TMP14:%.*]] = load float, ptr [[SRC_2]], align 4 | |
| ; DEFAULT-NEXT: [[TMP15:%.*]] = fmul float [[TMP14]], 0.000000e+00 | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <8 x float> poison, float [[TMP15]], i64 0 | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <8 x float> [[BROADCAST_SPLATINSERT2]], <8 x float> poison, <8 x i32> zeroinitializer | |
| ; DEFAULT-NEXT: [[TMP16:%.*]] = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> [[BROADCAST_SPLAT]], <8 x float> zeroinitializer, <8 x float> [[BROADCAST_SPLAT3]]) | |
| ; DEFAULT-NEXT: [[TMP17:%.*]] = load float, ptr [[SRC_3]], align 4 | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <8 x float> poison, float [[TMP17]], i64 0 | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <8 x float> [[BROADCAST_SPLATINSERT4]], <8 x float> poison, <8 x i32> zeroinitializer | |
| ; DEFAULT-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> [[BROADCAST_SPLAT5]], <8 x float> zeroinitializer, <8 x float> [[TMP16]]) | |
| ; DEFAULT-NEXT: [[TMP19:%.*]] = fcmp ogt <8 x float> [[TMP18]], [[BROADCAST_SPLAT5]] | |
| ; DEFAULT-NEXT: [[WIDE_GEP:%.*]] = getelementptr [16 x i8], ptr [[DST]], <8 x i64> [[VEC_IND]] | |
| ; DEFAULT-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[TMP19]], i64 0 | |
| ; DEFAULT-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] | |
| ; DEFAULT: [[PRED_STORE_IF]]: | |
| ; DEFAULT-NEXT: [[TMP21:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 0 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP21]], align 4 | |
| ; DEFAULT-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[TMP21]], i64 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP22]], align 4 | |
| ; DEFAULT-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[TMP21]], i64 8 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP23]], align 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP21]], align 4 | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE]]: | |
| ; DEFAULT-NEXT: [[TMP24:%.*]] = extractelement <8 x i1> [[TMP19]], i64 1 | |
| ; DEFAULT-NEXT: br i1 [[TMP24]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]] | |
| ; DEFAULT: [[PRED_STORE_IF6]]: | |
| ; DEFAULT-NEXT: [[TMP25:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 1 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP25]], align 4 | |
| ; DEFAULT-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[TMP25]], i64 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP26]], align 4 | |
| ; DEFAULT-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[TMP25]], i64 8 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP27]], align 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP25]], align 4 | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE7]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE7]]: | |
| ; DEFAULT-NEXT: [[TMP28:%.*]] = extractelement <8 x i1> [[TMP19]], i64 2 | |
| ; DEFAULT-NEXT: br i1 [[TMP28]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]] | |
| ; DEFAULT: [[PRED_STORE_IF8]]: | |
| ; DEFAULT-NEXT: [[TMP29:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 2 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP29]], align 4 | |
| ; DEFAULT-NEXT: [[TMP30:%.*]] = getelementptr i8, ptr [[TMP29]], i64 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP30]], align 4 | |
| ; DEFAULT-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[TMP29]], i64 8 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP31]], align 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP29]], align 4 | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE9]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE9]]: | |
| ; DEFAULT-NEXT: [[TMP32:%.*]] = extractelement <8 x i1> [[TMP19]], i64 3 | |
| ; DEFAULT-NEXT: br i1 [[TMP32]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]] | |
| ; DEFAULT: [[PRED_STORE_IF10]]: | |
| ; DEFAULT-NEXT: [[TMP33:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 3 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP33]], align 4 | |
| ; DEFAULT-NEXT: [[TMP34:%.*]] = getelementptr i8, ptr [[TMP33]], i64 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP34]], align 4 | |
| ; DEFAULT-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[TMP33]], i64 8 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP35]], align 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP33]], align 4 | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE11]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE11]]: | |
| ; DEFAULT-NEXT: [[TMP36:%.*]] = extractelement <8 x i1> [[TMP19]], i64 4 | |
| ; DEFAULT-NEXT: br i1 [[TMP36]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]] | |
| ; DEFAULT: [[PRED_STORE_IF12]]: | |
| ; DEFAULT-NEXT: [[TMP37:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP37]], align 4 | |
| ; DEFAULT-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[TMP37]], i64 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP38]], align 4 | |
| ; DEFAULT-NEXT: [[TMP39:%.*]] = getelementptr i8, ptr [[TMP37]], i64 8 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP39]], align 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP37]], align 4 | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE13]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE13]]: | |
| ; DEFAULT-NEXT: [[TMP40:%.*]] = extractelement <8 x i1> [[TMP19]], i64 5 | |
| ; DEFAULT-NEXT: br i1 [[TMP40]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15:.*]] | |
| ; DEFAULT: [[PRED_STORE_IF14]]: | |
| ; DEFAULT-NEXT: [[TMP41:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 5 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP41]], align 4 | |
| ; DEFAULT-NEXT: [[TMP42:%.*]] = getelementptr i8, ptr [[TMP41]], i64 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP42]], align 4 | |
| ; DEFAULT-NEXT: [[TMP43:%.*]] = getelementptr i8, ptr [[TMP41]], i64 8 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP43]], align 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP41]], align 4 | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE15]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE15]]: | |
| ; DEFAULT-NEXT: [[TMP44:%.*]] = extractelement <8 x i1> [[TMP19]], i64 6 | |
| ; DEFAULT-NEXT: br i1 [[TMP44]], label %[[PRED_STORE_IF16:.*]], label %[[PRED_STORE_CONTINUE17:.*]] | |
| ; DEFAULT: [[PRED_STORE_IF16]]: | |
| ; DEFAULT-NEXT: [[TMP45:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 6 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP45]], align 4 | |
| ; DEFAULT-NEXT: [[TMP46:%.*]] = getelementptr i8, ptr [[TMP45]], i64 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP46]], align 4 | |
| ; DEFAULT-NEXT: [[TMP47:%.*]] = getelementptr i8, ptr [[TMP45]], i64 8 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP47]], align 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP45]], align 4 | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE17]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE17]]: | |
| ; DEFAULT-NEXT: [[TMP48:%.*]] = extractelement <8 x i1> [[TMP19]], i64 7 | |
| ; DEFAULT-NEXT: br i1 [[TMP48]], label %[[PRED_STORE_IF18:.*]], label %[[PRED_STORE_CONTINUE19]] | |
| ; DEFAULT: [[PRED_STORE_IF18]]: | |
| ; DEFAULT-NEXT: [[TMP49:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 7 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP49]], align 4 | |
| ; DEFAULT-NEXT: [[TMP50:%.*]] = getelementptr i8, ptr [[TMP49]], i64 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP50]], align 4 | |
| ; DEFAULT-NEXT: [[TMP51:%.*]] = getelementptr i8, ptr [[TMP49]], i64 8 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP51]], align 4 | |
| ; DEFAULT-NEXT: store float 0.000000e+00, ptr [[TMP49]], align 4 | |
| ; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE19]] | |
| ; DEFAULT: [[PRED_STORE_CONTINUE19]]: | |
| ; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 | |
| ; DEFAULT-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8) | |
| ; DEFAULT-NEXT: [[TMP52:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; DEFAULT-NEXT: br i1 [[TMP52]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] | |
| ; DEFAULT: [[MIDDLE_BLOCK]]: | |
| ; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] | |
| ; DEFAULT-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] | |
| ; DEFAULT: [[SCALAR_PH]]: | |
| ; | |
| ; PRED-LABEL: define void @test_conditional_interleave_group( | |
| ; PRED-SAME: ptr noalias [[SRC_1:%.*]], ptr noalias [[SRC_2:%.*]], ptr noalias [[SRC_3:%.*]], ptr noalias [[SRC_4:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) #[[ATTR2:[0-9]+]] { | |
| ; PRED-NEXT: [[ENTRY:.*:]] | |
| ; PRED-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 | |
| ; PRED-NEXT: br label %[[VECTOR_SCEVCHECK:.*]] | |
| ; PRED: [[VECTOR_SCEVCHECK]]: | |
| ; PRED-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]]) | |
| ; PRED-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0 | |
| ; PRED-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1 | |
| ; PRED-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]] | |
| ; PRED-NEXT: [[TMP2:%.*]] = icmp ult ptr [[TMP1]], [[DST]] | |
| ; PRED-NEXT: [[TMP3:%.*]] = or i1 [[TMP2]], [[MUL_OVERFLOW]] | |
| ; PRED-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4 | |
| ; PRED-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]] | |
| ; PRED-NEXT: [[TMP5:%.*]] = icmp ult ptr [[TMP4]], [[SCEVGEP]] | |
| ; PRED-NEXT: [[TMP6:%.*]] = or i1 [[TMP5]], [[MUL_OVERFLOW]] | |
| ; PRED-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[DST]], i64 8 | |
| ; PRED-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[MUL_RESULT]] | |
| ; PRED-NEXT: [[TMP8:%.*]] = icmp ult ptr [[TMP7]], [[SCEVGEP1]] | |
| ; PRED-NEXT: [[TMP9:%.*]] = or i1 [[TMP8]], [[MUL_OVERFLOW]] | |
| ; PRED-NEXT: [[TMP10:%.*]] = or i1 [[TMP3]], [[TMP6]] | |
| ; PRED-NEXT: [[TMP11:%.*]] = or i1 [[TMP10]], [[TMP9]] | |
| ; PRED-NEXT: br i1 [[TMP11]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; PRED: [[VECTOR_PH]]: | |
| ; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 [[TMP0]]) | |
| ; PRED-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[TMP0]], i64 8) | |
| ; PRED-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; PRED: [[VECTOR_BODY]]: | |
| ; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE19:.*]] ] | |
| ; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE19]] ] | |
| ; PRED-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE19]] ] | |
| ; PRED-NEXT: [[TMP12:%.*]] = load float, ptr [[SRC_1]], align 4 | |
| ; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x float> poison, float [[TMP12]], i64 0 | |
| ; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x float> [[BROADCAST_SPLATINSERT]], <8 x float> poison, <8 x i32> zeroinitializer | |
| ; PRED-NEXT: [[TMP13:%.*]] = load float, ptr [[SRC_2]], align 4 | |
| ; PRED-NEXT: [[TMP14:%.*]] = fmul float [[TMP13]], 0.000000e+00 | |
| ; PRED-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <8 x float> poison, float [[TMP14]], i64 0 | |
| ; PRED-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <8 x float> [[BROADCAST_SPLATINSERT2]], <8 x float> poison, <8 x i32> zeroinitializer | |
| ; PRED-NEXT: [[TMP15:%.*]] = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> [[BROADCAST_SPLAT]], <8 x float> zeroinitializer, <8 x float> [[BROADCAST_SPLAT3]]) | |
| ; PRED-NEXT: [[TMP16:%.*]] = load float, ptr [[SRC_3]], align 4 | |
| ; PRED-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <8 x float> poison, float [[TMP16]], i64 0 | |
| ; PRED-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <8 x float> [[BROADCAST_SPLATINSERT4]], <8 x float> poison, <8 x i32> zeroinitializer | |
| ; PRED-NEXT: [[TMP17:%.*]] = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> [[BROADCAST_SPLAT5]], <8 x float> zeroinitializer, <8 x float> [[TMP15]]) | |
| ; PRED-NEXT: [[TMP18:%.*]] = fcmp ogt <8 x float> [[TMP17]], [[BROADCAST_SPLAT5]] | |
| ; PRED-NEXT: [[TMP19:%.*]] = select <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i1> [[TMP18]], <8 x i1> zeroinitializer | |
| ; PRED-NEXT: [[WIDE_GEP:%.*]] = getelementptr [16 x i8], ptr [[DST]], <8 x i64> [[VEC_IND]] | |
| ; PRED-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[TMP19]], i64 0 | |
| ; PRED-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] | |
| ; PRED: [[PRED_STORE_IF]]: | |
| ; PRED-NEXT: [[TMP21:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 0 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP21]], align 4 | |
| ; PRED-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[TMP21]], i64 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP22]], align 4 | |
| ; PRED-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[TMP21]], i64 8 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP23]], align 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP21]], align 4 | |
| ; PRED-NEXT: br label %[[PRED_STORE_CONTINUE]] | |
| ; PRED: [[PRED_STORE_CONTINUE]]: | |
| ; PRED-NEXT: [[TMP24:%.*]] = extractelement <8 x i1> [[TMP19]], i64 1 | |
| ; PRED-NEXT: br i1 [[TMP24]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]] | |
| ; PRED: [[PRED_STORE_IF6]]: | |
| ; PRED-NEXT: [[TMP25:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 1 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP25]], align 4 | |
| ; PRED-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[TMP25]], i64 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP26]], align 4 | |
| ; PRED-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[TMP25]], i64 8 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP27]], align 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP25]], align 4 | |
| ; PRED-NEXT: br label %[[PRED_STORE_CONTINUE7]] | |
| ; PRED: [[PRED_STORE_CONTINUE7]]: | |
| ; PRED-NEXT: [[TMP28:%.*]] = extractelement <8 x i1> [[TMP19]], i64 2 | |
| ; PRED-NEXT: br i1 [[TMP28]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]] | |
| ; PRED: [[PRED_STORE_IF8]]: | |
| ; PRED-NEXT: [[TMP29:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 2 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP29]], align 4 | |
| ; PRED-NEXT: [[TMP30:%.*]] = getelementptr i8, ptr [[TMP29]], i64 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP30]], align 4 | |
| ; PRED-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[TMP29]], i64 8 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP31]], align 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP29]], align 4 | |
| ; PRED-NEXT: br label %[[PRED_STORE_CONTINUE9]] | |
| ; PRED: [[PRED_STORE_CONTINUE9]]: | |
| ; PRED-NEXT: [[TMP32:%.*]] = extractelement <8 x i1> [[TMP19]], i64 3 | |
| ; PRED-NEXT: br i1 [[TMP32]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]] | |
| ; PRED: [[PRED_STORE_IF10]]: | |
| ; PRED-NEXT: [[TMP33:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 3 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP33]], align 4 | |
| ; PRED-NEXT: [[TMP34:%.*]] = getelementptr i8, ptr [[TMP33]], i64 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP34]], align 4 | |
| ; PRED-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[TMP33]], i64 8 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP35]], align 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP33]], align 4 | |
| ; PRED-NEXT: br label %[[PRED_STORE_CONTINUE11]] | |
| ; PRED: [[PRED_STORE_CONTINUE11]]: | |
| ; PRED-NEXT: [[TMP36:%.*]] = extractelement <8 x i1> [[TMP19]], i64 4 | |
| ; PRED-NEXT: br i1 [[TMP36]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]] | |
| ; PRED: [[PRED_STORE_IF12]]: | |
| ; PRED-NEXT: [[TMP37:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP37]], align 4 | |
| ; PRED-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[TMP37]], i64 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP38]], align 4 | |
| ; PRED-NEXT: [[TMP39:%.*]] = getelementptr i8, ptr [[TMP37]], i64 8 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP39]], align 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP37]], align 4 | |
| ; PRED-NEXT: br label %[[PRED_STORE_CONTINUE13]] | |
| ; PRED: [[PRED_STORE_CONTINUE13]]: | |
| ; PRED-NEXT: [[TMP40:%.*]] = extractelement <8 x i1> [[TMP19]], i64 5 | |
| ; PRED-NEXT: br i1 [[TMP40]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15:.*]] | |
| ; PRED: [[PRED_STORE_IF14]]: | |
| ; PRED-NEXT: [[TMP41:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 5 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP41]], align 4 | |
| ; PRED-NEXT: [[TMP42:%.*]] = getelementptr i8, ptr [[TMP41]], i64 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP42]], align 4 | |
| ; PRED-NEXT: [[TMP43:%.*]] = getelementptr i8, ptr [[TMP41]], i64 8 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP43]], align 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP41]], align 4 | |
| ; PRED-NEXT: br label %[[PRED_STORE_CONTINUE15]] | |
| ; PRED: [[PRED_STORE_CONTINUE15]]: | |
| ; PRED-NEXT: [[TMP44:%.*]] = extractelement <8 x i1> [[TMP19]], i64 6 | |
| ; PRED-NEXT: br i1 [[TMP44]], label %[[PRED_STORE_IF16:.*]], label %[[PRED_STORE_CONTINUE17:.*]] | |
| ; PRED: [[PRED_STORE_IF16]]: | |
| ; PRED-NEXT: [[TMP45:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 6 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP45]], align 4 | |
| ; PRED-NEXT: [[TMP46:%.*]] = getelementptr i8, ptr [[TMP45]], i64 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP46]], align 4 | |
| ; PRED-NEXT: [[TMP47:%.*]] = getelementptr i8, ptr [[TMP45]], i64 8 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP47]], align 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP45]], align 4 | |
| ; PRED-NEXT: br label %[[PRED_STORE_CONTINUE17]] | |
| ; PRED: [[PRED_STORE_CONTINUE17]]: | |
| ; PRED-NEXT: [[TMP48:%.*]] = extractelement <8 x i1> [[TMP19]], i64 7 | |
| ; PRED-NEXT: br i1 [[TMP48]], label %[[PRED_STORE_IF18:.*]], label %[[PRED_STORE_CONTINUE19]] | |
| ; PRED: [[PRED_STORE_IF18]]: | |
| ; PRED-NEXT: [[TMP49:%.*]] = extractelement <8 x ptr> [[WIDE_GEP]], i64 7 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP49]], align 4 | |
| ; PRED-NEXT: [[TMP50:%.*]] = getelementptr i8, ptr [[TMP49]], i64 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP50]], align 4 | |
| ; PRED-NEXT: [[TMP51:%.*]] = getelementptr i8, ptr [[TMP49]], i64 8 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP51]], align 4 | |
| ; PRED-NEXT: store float 0.000000e+00, ptr [[TMP49]], align 4 | |
| ; PRED-NEXT: br label %[[PRED_STORE_CONTINUE19]] | |
| ; PRED: [[PRED_STORE_CONTINUE19]]: | |
| ; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX]], i64 [[TRIP_COUNT_MINUS_VF]]) | |
| ; PRED-NEXT: [[TMP52:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0 | |
| ; PRED-NEXT: [[TMP53:%.*]] = xor i1 [[TMP52]], true | |
| ; PRED-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8) | |
| ; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8 | |
| ; PRED-NEXT: br i1 [[TMP53]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] | |
| ; PRED: [[MIDDLE_BLOCK]]: | |
| ; PRED-NEXT: br [[EXIT:label %.*]] | |
| ; PRED: [[SCALAR_PH]]: | |
| ; | |
| entry: | |
| br label %loop.header | |
| loop.header: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] | |
| %0 = load float, ptr %src.1, align 4 | |
| %1 = load float, ptr %src.2, align 4 | |
| %mul8.i.us = fmul float %1, 0.000000e+00 | |
| %2 = tail call float @llvm.fmuladd.f32(float %0, float 0.000000e+00, float %mul8.i.us) | |
| %3 = load float, ptr %src.3, align 4 | |
| %4 = tail call float @llvm.fmuladd.f32(float %3, float 0.000000e+00, float %2) | |
| %5 = load float, ptr %src.3, align 4 | |
| %c = fcmp ogt float %4, %5 | |
| br i1 %c, label %if.then, label %loop.latch | |
| if.then: | |
| %dst.0 = getelementptr { [4 x float] }, ptr %dst, i64 %iv | |
| store float 0.000000e+00, ptr %dst.0, align 4 | |
| %dst.1 = getelementptr i8, ptr %dst.0, i64 4 | |
| store float 0.000000e+00, ptr %dst.1, align 4 | |
| %dst.2 = getelementptr i8, ptr %dst.0, i64 8 | |
| store float 0.000000e+00, ptr %dst.2, align 4 | |
| %dst.3 = getelementptr i8, ptr %dst.0, i64 16 | |
| store float 0.000000e+00, ptr %dst.0, align 4 | |
| br label %loop.latch | |
| loop.latch: | |
| %iv.next = add i64 %iv, 1 | |
| %ec = icmp eq i64 %iv, %N | |
| br i1 %ec, label %exit, label %loop.header, !llvm.loop !0 | |
| exit: | |
| ret void | |
| } | |
| define void @redundant_branch_and_tail_folding(ptr %dst, i1 %c) { | |
| ; DEFAULT-LABEL: define void @redundant_branch_and_tail_folding( | |
| ; DEFAULT-SAME: ptr [[DST:%.*]], i1 [[C:%.*]]) { | |
| ; DEFAULT-NEXT: [[ENTRY:.*:]] | |
| ; DEFAULT-NEXT: br label %[[VECTOR_PH:.*]] | |
| ; DEFAULT: [[VECTOR_PH]]: | |
| ; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; DEFAULT: [[VECTOR_BODY]]: | |
| ; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; DEFAULT-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; DEFAULT-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4) | |
| ; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 | |
| ; DEFAULT-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[STEP_ADD]], splat (i64 4) | |
| ; DEFAULT-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 | |
| ; DEFAULT-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]] | |
| ; DEFAULT: [[MIDDLE_BLOCK]]: | |
| ; DEFAULT-NEXT: [[TMP1:%.*]] = add nuw nsw <4 x i64> [[STEP_ADD]], splat (i64 1) | |
| ; DEFAULT-NEXT: [[TMP2:%.*]] = trunc nuw nsw <4 x i64> [[TMP1]] to <4 x i32> | |
| ; DEFAULT-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP2]], i64 3 | |
| ; DEFAULT-NEXT: store i32 [[TMP3]], ptr [[DST]], align 4 | |
| ; DEFAULT-NEXT: br label %[[SCALAR_PH:.*]] | |
| ; DEFAULT: [[SCALAR_PH]]: | |
| ; | |
| ; PRED-LABEL: define void @redundant_branch_and_tail_folding( | |
| ; PRED-SAME: ptr [[DST:%.*]], i1 [[C:%.*]]) { | |
| ; PRED-NEXT: [[ENTRY:.*]]: | |
| ; PRED-NEXT: br label %[[LOOP_HEADER:.*]] | |
| ; PRED: [[LOOP_HEADER]]: | |
| ; PRED-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] | |
| ; PRED-NEXT: br i1 [[C]], label %[[LOOP_LATCH]], label %[[THEN:.*]] | |
| ; PRED: [[THEN]]: | |
| ; PRED-NEXT: br label %[[LOOP_LATCH]] | |
| ; PRED: [[LOOP_LATCH]]: | |
| ; PRED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 | |
| ; PRED-NEXT: [[T:%.*]] = trunc nuw nsw i64 [[IV_NEXT]] to i32 | |
| ; PRED-NEXT: store i32 [[T]], ptr [[DST]], align 4 | |
| ; PRED-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 21 | |
| ; PRED-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]] | |
| ; PRED: [[EXIT]]: | |
| ; PRED-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop.header | |
| loop.header: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] | |
| br i1 %c, label %loop.latch, label %then | |
| then: | |
| br label %loop.latch | |
| loop.latch: | |
| %iv.next = add nuw nsw i64 %iv, 1 | |
| %t = trunc nuw nsw i64 %iv.next to i32 | |
| store i32 %t, ptr %dst, align 4 | |
| %ec = icmp eq i64 %iv.next, 21 | |
| br i1 %ec, label %exit, label %loop.header | |
| exit: | |
| ret void | |
| } | |
| define double @test_cond_intrinsics(ptr %A, i1 %c) { | |
| ; COMMON-LABEL: define double @test_cond_intrinsics( | |
| ; COMMON-SAME: ptr [[A:%.*]], i1 [[C:%.*]]) { | |
| ; COMMON-NEXT: [[ENTRY:.*]]: | |
| ; COMMON-NEXT: br label %[[LOOP_HEADER:.*]] | |
| ; COMMON: [[LOOP_HEADER]]: | |
| ; COMMON-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] | |
| ; COMMON-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]] | |
| ; COMMON: [[THEN]]: | |
| ; COMMON-NEXT: [[GEP:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]] | |
| ; COMMON-NEXT: [[L:%.*]] = load double, ptr [[GEP]], align 8 | |
| ; COMMON-NEXT: [[POW:%.*]] = tail call double @llvm.pow.f64(double 2.000000e+00, double [[L]]) | |
| ; COMMON-NEXT: store double [[POW]], ptr [[GEP]], align 8 | |
| ; COMMON-NEXT: br label %[[LOOP_LATCH]] | |
| ; COMMON: [[LOOP_LATCH]]: | |
| ; COMMON-NEXT: [[P:%.*]] = phi double [ 0.000000e+00, %[[LOOP_HEADER]] ], [ [[POW]], %[[THEN]] ] | |
| ; COMMON-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 | |
| ; COMMON-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 1024 | |
| ; COMMON-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]] | |
| ; COMMON: [[EXIT]]: | |
| ; COMMON-NEXT: [[P_LCSSA:%.*]] = phi double [ [[P]], %[[LOOP_LATCH]] ] | |
| ; COMMON-NEXT: ret double [[P_LCSSA]] | |
| ; | |
| entry: | |
| br label %loop.header | |
| loop.header: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] | |
| br i1 %c, label %then, label %loop.latch | |
| then: | |
| %gep = getelementptr inbounds double, ptr %A, i64 %iv | |
| %l = load double, ptr %gep, align 8 | |
| %pow = tail call double @llvm.pow.f64(double 2.0, double %l) | |
| store double %pow, ptr %gep | |
| br label %loop.latch | |
| loop.latch: | |
| %p = phi double [ 0.000000e+00, %loop.header ], [ %pow, %then ] | |
| %iv.next = add i64 %iv, 1 | |
| %ec = icmp eq i64 %iv, 1024 | |
| br i1 %ec, label %exit, label %loop.header | |
| exit: | |
| ret double %p | |
| } | |
| define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 { | |
| ; DEFAULT-LABEL: define void @pred_udiv_select_cost( | |
| ; DEFAULT-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]], i8 [[Y:%.*]]) #[[ATTR1]] { | |
| ; DEFAULT-NEXT: [[ITER_CHECK:.*:]] | |
| ; DEFAULT-NEXT: [[C7:%.*]] = ptrtoaddr ptr [[C]] to i64 | |
| ; DEFAULT-NEXT: [[A6:%.*]] = ptrtoaddr ptr [[A]] to i64 | |
| ; DEFAULT-NEXT: [[B5:%.*]] = ptrtoaddr ptr [[B]] to i64 | |
| ; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 | |
| ; DEFAULT-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() | |
| ; DEFAULT-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 3 | |
| ; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]] | |
| ; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] | |
| ; DEFAULT: [[VECTOR_MEMCHECK]]: | |
| ; DEFAULT-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 4 | |
| ; DEFAULT-NEXT: [[TMP4:%.*]] = add nsw i64 [[TMP3]], -1 | |
| ; DEFAULT-NEXT: [[TMP5:%.*]] = sub i64 [[C7]], [[A6]] | |
| ; DEFAULT-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], 1 | |
| ; DEFAULT-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP4]] | |
| ; DEFAULT-NEXT: [[TMP7:%.*]] = sub i64 [[C7]], [[B5]] | |
| ; DEFAULT-NEXT: [[TMP8:%.*]] = sub i64 [[TMP7]], 1 | |
| ; DEFAULT-NEXT: [[DIFF_CHECK8:%.*]] = icmp ult i64 [[TMP8]], [[TMP4]] | |
| ; DEFAULT-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK8]] | |
| ; DEFAULT-NEXT: br i1 [[CONFLICT_RDX]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] | |
| ; DEFAULT: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: | |
| ; DEFAULT-NEXT: [[MIN_ITERS_CHECK9:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]] | |
| ; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK9]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; DEFAULT: [[VECTOR_PH]]: | |
| ; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP3]] | |
| ; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]] | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[Y]], i64 0 | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer | |
| ; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; DEFAULT: [[VECTOR_BODY]]: | |
| ; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; DEFAULT-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]] | |
| ; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP9]], align 1 | |
| ; DEFAULT-NEXT: [[TMP10:%.*]] = uitofp <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x float> | |
| ; DEFAULT-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]] | |
| ; DEFAULT-NEXT: [[WIDE_LOAD10:%.*]] = load <vscale x 16 x i8>, ptr [[TMP11]], align 1 | |
| ; DEFAULT-NEXT: [[TMP12:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD10]], zeroinitializer | |
| ; DEFAULT-NEXT: [[TMP13:%.*]] = xor <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 1) | |
| ; DEFAULT-NEXT: [[TMP14:%.*]] = call <vscale x 16 x i8> @llvm.masked.udiv.nxv16i8(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[BROADCAST_SPLAT]], <vscale x 16 x i1> [[TMP12]]) | |
| ; DEFAULT-NEXT: [[TMP15:%.*]] = icmp ugt <vscale x 16 x i8> [[TMP14]], splat (i8 1) | |
| ; DEFAULT-NEXT: [[TMP16:%.*]] = select <vscale x 16 x i1> [[TMP15]], <vscale x 16 x i32> zeroinitializer, <vscale x 16 x i32> splat (i32 255) | |
| ; DEFAULT-NEXT: [[PREDPHI:%.*]] = select <vscale x 16 x i1> [[TMP12]], <vscale x 16 x i32> [[TMP16]], <vscale x 16 x i32> zeroinitializer | |
| ; DEFAULT-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32> | |
| ; DEFAULT-NEXT: [[TMP18:%.*]] = sub <vscale x 16 x i32> [[PREDPHI]], [[TMP17]] | |
| ; DEFAULT-NEXT: [[TMP19:%.*]] = sitofp <vscale x 16 x i32> [[TMP18]] to <vscale x 16 x float> | |
| ; DEFAULT-NEXT: [[TMP20:%.*]] = call <vscale x 16 x float> @llvm.fmuladd.nxv16f32(<vscale x 16 x float> [[TMP19]], <vscale x 16 x float> splat (float 3.000000e+00), <vscale x 16 x float> [[TMP10]]) | |
| ; DEFAULT-NEXT: [[TMP21:%.*]] = fptoui <vscale x 16 x float> [[TMP20]] to <vscale x 16 x i8> | |
| ; DEFAULT-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]] | |
| ; DEFAULT-NEXT: store <vscale x 16 x i8> [[TMP21]], ptr [[TMP22]], align 1 | |
| ; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] | |
| ; DEFAULT-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; DEFAULT-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]] | |
| ; DEFAULT: [[MIDDLE_BLOCK]]: | |
| ; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] | |
| ; DEFAULT-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] | |
| ; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]: | |
| ; DEFAULT-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP2]] | |
| ; DEFAULT-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF30:![0-9]+]] | |
| ; DEFAULT: [[VEC_EPILOG_PH]]: | |
| ; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] | |
| ; DEFAULT-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64() | |
| ; DEFAULT-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 3 | |
| ; DEFAULT-NEXT: [[N_MOD_VF11:%.*]] = urem i64 [[TMP0]], [[TMP25]] | |
| ; DEFAULT-NEXT: [[N_VEC12:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF11]] | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[Y]], i64 0 | |
| ; DEFAULT-NEXT: [[BROADCAST_SPLAT14:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT13]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer | |
| ; DEFAULT-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]] | |
| ; DEFAULT: [[VEC_EPILOG_VECTOR_BODY]]: | |
| ; DEFAULT-NEXT: [[INDEX15:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT19:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ] | |
| ; DEFAULT-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX15]] | |
| ; DEFAULT-NEXT: [[WIDE_LOAD16:%.*]] = load <vscale x 8 x i8>, ptr [[TMP26]], align 1 | |
| ; DEFAULT-NEXT: [[TMP27:%.*]] = uitofp <vscale x 8 x i8> [[WIDE_LOAD16]] to <vscale x 8 x float> | |
| ; DEFAULT-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX15]] | |
| ; DEFAULT-NEXT: [[WIDE_LOAD17:%.*]] = load <vscale x 8 x i8>, ptr [[TMP28]], align 1 | |
| ; DEFAULT-NEXT: [[TMP29:%.*]] = icmp ne <vscale x 8 x i8> [[WIDE_LOAD17]], zeroinitializer | |
| ; DEFAULT-NEXT: [[TMP30:%.*]] = xor <vscale x 8 x i8> [[WIDE_LOAD16]], splat (i8 1) | |
| ; DEFAULT-NEXT: [[TMP31:%.*]] = call <vscale x 8 x i8> @llvm.masked.udiv.nxv8i8(<vscale x 8 x i8> [[TMP30]], <vscale x 8 x i8> [[BROADCAST_SPLAT14]], <vscale x 8 x i1> [[TMP29]]) | |
| ; DEFAULT-NEXT: [[TMP32:%.*]] = icmp ugt <vscale x 8 x i8> [[TMP31]], splat (i8 1) | |
| ; DEFAULT-NEXT: [[TMP33:%.*]] = select <vscale x 8 x i1> [[TMP32]], <vscale x 8 x i32> zeroinitializer, <vscale x 8 x i32> splat (i32 255) | |
| ; DEFAULT-NEXT: [[PREDPHI18:%.*]] = select <vscale x 8 x i1> [[TMP29]], <vscale x 8 x i32> [[TMP33]], <vscale x 8 x i32> zeroinitializer | |
| ; DEFAULT-NEXT: [[TMP34:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD16]] to <vscale x 8 x i32> | |
| ; DEFAULT-NEXT: [[TMP35:%.*]] = sub <vscale x 8 x i32> [[PREDPHI18]], [[TMP34]] | |
| ; DEFAULT-NEXT: [[TMP36:%.*]] = sitofp <vscale x 8 x i32> [[TMP35]] to <vscale x 8 x float> | |
| ; DEFAULT-NEXT: [[TMP37:%.*]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[TMP36]], <vscale x 8 x float> splat (float 3.000000e+00), <vscale x 8 x float> [[TMP27]]) | |
| ; DEFAULT-NEXT: [[TMP38:%.*]] = fptoui <vscale x 8 x float> [[TMP37]] to <vscale x 8 x i8> | |
| ; DEFAULT-NEXT: [[TMP39:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX15]] | |
| ; DEFAULT-NEXT: store <vscale x 8 x i8> [[TMP38]], ptr [[TMP39]], align 1 | |
| ; DEFAULT-NEXT: [[INDEX_NEXT19]] = add nuw i64 [[INDEX15]], [[TMP25]] | |
| ; DEFAULT-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT19]], [[N_VEC12]] | |
| ; DEFAULT-NEXT: br i1 [[TMP40]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]] | |
| ; DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]: | |
| ; DEFAULT-NEXT: [[CMP_N20:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC12]] | |
| ; DEFAULT-NEXT: br i1 [[CMP_N20]], [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]] | |
| ; DEFAULT: [[VEC_EPILOG_SCALAR_PH]]: | |
| ; | |
| ; PRED-LABEL: define void @pred_udiv_select_cost( | |
| ; PRED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]], i8 [[Y:%.*]]) #[[ATTR1]] { | |
| ; PRED-NEXT: [[ENTRY:.*:]] | |
| ; PRED-NEXT: [[B7:%.*]] = ptrtoaddr ptr [[B]] to i64 | |
| ; PRED-NEXT: [[A6:%.*]] = ptrtoaddr ptr [[A]] to i64 | |
| ; PRED-NEXT: [[C5:%.*]] = ptrtoaddr ptr [[C]] to i64 | |
| ; PRED-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 | |
| ; PRED-NEXT: br label %[[VECTOR_MEMCHECK:.*]] | |
| ; PRED: [[VECTOR_MEMCHECK]]: | |
| ; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() | |
| ; PRED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4 | |
| ; PRED-NEXT: [[TMP3:%.*]] = add nsw i64 [[TMP2]], -1 | |
| ; PRED-NEXT: [[TMP4:%.*]] = sub i64 [[C5]], [[A6]] | |
| ; PRED-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 1 | |
| ; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], [[TMP3]] | |
| ; PRED-NEXT: [[TMP6:%.*]] = sub i64 [[C5]], [[B7]] | |
| ; PRED-NEXT: [[TMP7:%.*]] = sub i64 [[TMP6]], 1 | |
| ; PRED-NEXT: [[DIFF_CHECK8:%.*]] = icmp ult i64 [[TMP7]], [[TMP3]] | |
| ; PRED-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK8]] | |
| ; PRED-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; PRED: [[VECTOR_PH]]: | |
| ; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[TMP0]]) | |
| ; PRED-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[TMP0]], i64 [[TMP2]]) | |
| ; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[Y]], i64 0 | |
| ; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer | |
| ; PRED-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; PRED: [[VECTOR_BODY]]: | |
| ; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; PRED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]] | |
| ; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison) | |
| ; PRED-NEXT: [[TMP9:%.*]] = uitofp <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x float> | |
| ; PRED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]] | |
| ; PRED-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison) | |
| ; PRED-NEXT: [[TMP11:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD9]], zeroinitializer | |
| ; PRED-NEXT: [[TMP12:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP11]], <vscale x 16 x i1> zeroinitializer | |
| ; PRED-NEXT: [[TMP13:%.*]] = xor <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 1) | |
| ; PRED-NEXT: [[TMP14:%.*]] = call <vscale x 16 x i8> @llvm.masked.udiv.nxv16i8(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[BROADCAST_SPLAT]], <vscale x 16 x i1> [[TMP12]]) | |
| ; PRED-NEXT: [[TMP15:%.*]] = icmp ugt <vscale x 16 x i8> [[TMP14]], splat (i8 1) | |
| ; PRED-NEXT: [[TMP16:%.*]] = select <vscale x 16 x i1> [[TMP15]], <vscale x 16 x i32> zeroinitializer, <vscale x 16 x i32> splat (i32 255) | |
| ; PRED-NEXT: [[PREDPHI:%.*]] = select <vscale x 16 x i1> [[TMP11]], <vscale x 16 x i32> [[TMP16]], <vscale x 16 x i32> zeroinitializer | |
| ; PRED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32> | |
| ; PRED-NEXT: [[TMP18:%.*]] = sub <vscale x 16 x i32> [[PREDPHI]], [[TMP17]] | |
| ; PRED-NEXT: [[TMP19:%.*]] = sitofp <vscale x 16 x i32> [[TMP18]] to <vscale x 16 x float> | |
| ; PRED-NEXT: [[TMP20:%.*]] = call <vscale x 16 x float> @llvm.fmuladd.nxv16f32(<vscale x 16 x float> [[TMP19]], <vscale x 16 x float> splat (float 3.000000e+00), <vscale x 16 x float> [[TMP9]]) | |
| ; PRED-NEXT: [[TMP21:%.*]] = fptoui <vscale x 16 x float> [[TMP20]] to <vscale x 16 x i8> | |
| ; PRED-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]] | |
| ; PRED-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP21]], ptr align 1 [[TMP22]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]]) | |
| ; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX]], i64 [[TRIP_COUNT_MINUS_VF]]) | |
| ; PRED-NEXT: [[TMP23:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0 | |
| ; PRED-NEXT: [[TMP24:%.*]] = xor i1 [[TMP23]], true | |
| ; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]] | |
| ; PRED-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] | |
| ; PRED: [[MIDDLE_BLOCK]]: | |
| ; PRED-NEXT: br [[EXIT:label %.*]] | |
| ; PRED: [[SCALAR_PH]]: | |
| ; | |
| entry: | |
| br label %loop.header | |
| loop.header: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] | |
| %gep.A = getelementptr i8, ptr %A, i64 %iv | |
| %l.1 = load i8, ptr %gep.A, align 1 | |
| %l.1.to.float = uitofp i8 %l.1 to float | |
| %gep.B = getelementptr i8, ptr %B, i64 %iv | |
| %l.2 = load i8, ptr %gep.B, align 1 | |
| %c = icmp eq i8 %l.2, 0 | |
| br i1 %c, label %loop.latch, label %else | |
| else: | |
| %x = xor i8 %l.1, 1 | |
| %ud = udiv i8 %x, %y | |
| %c.sel = icmp ugt i8 %ud, 1 | |
| %sel = select i1 %c.sel, i32 0, i32 255 | |
| br label %loop.latch | |
| loop.latch: | |
| %p = phi i32 [ %sel, %else ], [ 0, %loop.header ] | |
| %l.1.ext = zext i8 %l.1 to i32 | |
| %sub = sub i32 %p, %l.1.ext | |
| %sub.to.float = sitofp i32 %sub to float | |
| %muladd = tail call float @llvm.fmuladd.f32(float %sub.to.float, float 3.000000e+00, float %l.1.to.float) | |
| %muladd.conv = fptoui float %muladd to i8 | |
| %gep.C = getelementptr i8, ptr %C, i64 %iv | |
| store i8 %muladd.conv, ptr %gep.C, align 1 | |
| %iv.next = add i64 %iv, 1 | |
| %ec = icmp eq i64 %iv, %n | |
| br i1 %ec, label %exit, label %loop.header | |
| exit: | |
| ret void | |
| } | |
| define void @predicated_store(ptr %A, ptr noalias %B, ptr noalias %C, ptr %D, ptr %E, double %divisor, i64 %loop.count) #3 { | |
| ; COMMON-LABEL: define void @predicated_store( | |
| ; COMMON-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], ptr [[D:%.*]], ptr [[E:%.*]], double [[DIVISOR:%.*]], i64 [[LOOP_COUNT:%.*]]) #[[ATTR3:[0-9]+]] { | |
| ; COMMON-NEXT: [[ENTRY:.*]]: | |
| ; COMMON-NEXT: br label %[[LOOP_HEADER:.*]] | |
| ; COMMON: [[LOOP_HEADER]]: | |
| ; COMMON-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] | |
| ; COMMON-NEXT: [[GEP_B:%.*]] = getelementptr double, ptr [[B]], i64 [[IV]] | |
| ; COMMON-NEXT: [[FIRST_VAL:%.*]] = load double, ptr [[B]], align 8 | |
| ; COMMON-NEXT: store double 0.000000e+00, ptr [[D]], align 8 | |
| ; COMMON-NEXT: [[COEFF:%.*]] = load double, ptr [[C]], align 8 | |
| ; COMMON-NEXT: [[NEG:%.*]] = fneg double [[FIRST_VAL]] | |
| ; COMMON-NEXT: [[NORM:%.*]] = fdiv double [[NEG]], [[DIVISOR]] | |
| ; COMMON-NEXT: [[SCALED_COEFF:%.*]] = fmul double [[COEFF]], [[NORM]] | |
| ; COMMON-NEXT: [[INPUT_VAL:%.*]] = load double, ptr [[GEP_B]], align 8 | |
| ; COMMON-NEXT: [[SCALED_INPUT:%.*]] = fmul double [[INPUT_VAL]], [[DIVISOR]] | |
| ; COMMON-NEXT: [[PRODUCT:%.*]] = fmul double [[SCALED_COEFF]], [[SCALED_INPUT]] | |
| ; COMMON-NEXT: [[FINAL:%.*]] = fmul double [[PRODUCT]], [[DIVISOR]] | |
| ; COMMON-NEXT: [[IS_POS:%.*]] = fcmp ogt double [[FINAL]], 0.000000e+00 | |
| ; COMMON-NEXT: br i1 [[IS_POS]], label %[[THEN:.*]], label %[[LOOP_LATCH]] | |
| ; COMMON: [[THEN]]: | |
| ; COMMON-NEXT: store double 0.000000e+00, ptr [[A]], align 8 | |
| ; COMMON-NEXT: [[GEP_E:%.*]] = getelementptr i8, ptr [[E]], i64 [[IV]] | |
| ; COMMON-NEXT: store double 0.000000e+00, ptr [[GEP_E]], align 8 | |
| ; COMMON-NEXT: br label %[[LOOP_LATCH]] | |
| ; COMMON: [[LOOP_LATCH]]: | |
| ; COMMON-NEXT: store double 0.000000e+00, ptr null, align 8 | |
| ; COMMON-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 | |
| ; COMMON-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[LOOP_COUNT]] | |
| ; COMMON-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]] | |
| ; COMMON: [[EXIT]]: | |
| ; COMMON-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop.header | |
| loop.header: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] | |
| %gep.B = getelementptr double, ptr %B, i64 %iv | |
| %first.val = load double, ptr %B, align 8 | |
| store double 0.000000e+00, ptr %D, align 8 | |
| %coeff = load double, ptr %C, align 8 | |
| %neg = fneg double %first.val | |
| %norm = fdiv double %neg, %divisor | |
| %scaled.coeff = fmul double %coeff, %norm | |
| %input.val = load double, ptr %gep.B, align 8 | |
| %scaled.input = fmul double %input.val, %divisor | |
| %product = fmul double %scaled.coeff, %scaled.input | |
| %final = fmul double %product, %divisor | |
| %is.pos = fcmp ogt double %final, 0.000000e+00 | |
| br i1 %is.pos, label %then, label %loop.latch | |
| then: | |
| store double 0.000000e+00, ptr %A, align 8 | |
| %gep.E = getelementptr i8, ptr %E, i64 %iv | |
| store double 0.000000e+00, ptr %gep.E, align 8 | |
| br label %loop.latch | |
| loop.latch: | |
| store double 0.000000e+00, ptr null, align 8 | |
| %iv.next = add i64 %iv, 1 | |
| %ec = icmp eq i64 %iv, %loop.count | |
| br i1 %ec, label %exit, label %loop.header | |
| exit: | |
| ret void | |
| } | |
| attributes #1 = { vscale_range(1,16) "target-cpu"="neoverse-512tvb" } | |
| attributes #2 = { vscale_range(2,2) "target-cpu"="neoverse-512tvb" } | |
| attributes #3 = { vscale_range(1,16) "target-cpu"="neoverse-v2" } | |
| !0 = distinct !{!0, !1, !2, !3} | |
| !1 = !{!"llvm.loop.vectorize.width", i32 8} | |
| !2 = !{!"llvm.loop.vectorize.scalable.disable"} | |
| !3 = !{!"llvm.loop.vectorize.enable"} |