| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6 |
| ; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -S | FileCheck %s --check-prefixes=VF2 |
| ; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -S | FileCheck %s --check-prefixes=VF8 |
| |
| ; Check that a conditionally-executed call with a vector library variant is |
| ; costed correctly when deciding whether scalarizing a predicated tree of |
| ; operations is profitable. Its result feeds a scatter store that must be |
| ; scalarized; the cost model uses the wide-call cost to decide whether to also |
| ; scalarize the call. At VF=2 scalarizing the call is cheapest, so it is |
| ; scalarized; at VF=8 the wide variant is cheaper, so the call stays wide and |
| ; only the store is scalarized. Querying the wide-call cost on this path |
| ; previously crashed. |
| |
| define void @pred_call_with_variant(ptr readonly %src, ptr noalias %dest, i64 %N) { |
| ; VF2-LABEL: define void @pred_call_with_variant( |
| ; VF2-SAME: ptr readonly [[SRC:%.*]], ptr noalias [[DEST:%.*]], i64 [[N:%.*]]) { |
| ; VF2-NEXT: [[ENTRY:.*]]: |
| ; VF2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2 |
| ; VF2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; VF2: [[VECTOR_PH]]: |
| ; VF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2 |
| ; VF2-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; VF2-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; VF2: [[VECTOR_BODY]]: |
| ; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ] |
| ; VF2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]] |
| ; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP0]], align 8 |
| ; VF2-NEXT: [[TMP1:%.*]] = icmp ult <2 x i64> [[WIDE_LOAD]], splat (i64 5) |
| ; VF2-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0 |
| ; VF2-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] |
| ; VF2: [[PRED_STORE_IF]]: |
| ; VF2-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 0 |
| ; VF2-NEXT: [[TMP4:%.*]] = call i64 @foo(i64 [[TMP3]]) #[[ATTR0:[0-9]+]] |
| ; VF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP3]] |
| ; VF2-NEXT: store i64 [[TMP4]], ptr [[TMP5]], align 8 |
| ; VF2-NEXT: br label %[[PRED_STORE_CONTINUE]] |
| ; VF2: [[PRED_STORE_CONTINUE]]: |
| ; VF2-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1 |
| ; VF2-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]] |
| ; VF2: [[PRED_STORE_IF1]]: |
| ; VF2-NEXT: [[TMP7:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 1 |
| ; VF2-NEXT: [[TMP8:%.*]] = call i64 @foo(i64 [[TMP7]]) #[[ATTR0]] |
| ; VF2-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP7]] |
| ; VF2-NEXT: store i64 [[TMP8]], ptr [[TMP9]], align 8 |
| ; VF2-NEXT: br label %[[PRED_STORE_CONTINUE2]] |
| ; VF2: [[PRED_STORE_CONTINUE2]]: |
| ; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; VF2-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; VF2-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; VF2: [[MIDDLE_BLOCK]]: |
| ; VF2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; VF2-NEXT: br i1 [[CMP_N]], label %[[END:.*]], label %[[SCALAR_PH]] |
| ; VF2: [[SCALAR_PH]]: |
| ; VF2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; VF2-NEXT: br label %[[FOR_BODY:.*]] |
| ; VF2: [[FOR_BODY]]: |
| ; VF2-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_LOOP:.*]] ] |
| ; VF2-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]] |
| ; VF2-NEXT: [[IDX:%.*]] = load i64, ptr [[LD_ADDR]], align 8 |
| ; VF2-NEXT: [[IFCOND:%.*]] = icmp ult i64 [[IDX]], 5 |
| ; VF2-NEXT: br i1 [[IFCOND]], label %[[IF_THEN:.*]], label %[[FOR_LOOP]] |
| ; VF2: [[IF_THEN]]: |
| ; VF2-NEXT: [[FOO_RET:%.*]] = call i64 @foo(i64 [[IDX]]) #[[ATTR0]] |
| ; VF2-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[IDX]] |
| ; VF2-NEXT: store i64 [[FOO_RET]], ptr [[ST_ADDR]], align 8 |
| ; VF2-NEXT: br label %[[FOR_LOOP]] |
| ; VF2: [[FOR_LOOP]]: |
| ; VF2-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 |
| ; VF2-NEXT: [[LOOPCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] |
| ; VF2-NEXT: br i1 [[LOOPCOND]], label %[[END]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] |
| ; VF2: [[END]]: |
| ; VF2-NEXT: ret void |
| ; |
| ; VF8-LABEL: define void @pred_call_with_variant( |
| ; VF8-SAME: ptr readonly [[SRC:%.*]], ptr noalias [[DEST:%.*]], i64 [[N:%.*]]) { |
| ; VF8-NEXT: [[ENTRY:.*]]: |
| ; VF8-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 |
| ; VF8-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; VF8: [[VECTOR_PH]]: |
| ; VF8-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8 |
| ; VF8-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; VF8-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; VF8: [[VECTOR_BODY]]: |
| ; VF8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE14:.*]] ] |
| ; VF8-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]] |
| ; VF8-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i64>, ptr [[TMP0]], align 8 |
| ; VF8-NEXT: [[TMP1:%.*]] = icmp ult <8 x i64> [[WIDE_LOAD]], splat (i64 5) |
| ; VF8-NEXT: [[TMP2:%.*]] = call <8 x i64> @vector_foo_8(<8 x i64> [[WIDE_LOAD]], <8 x i1> [[TMP1]]) |
| ; VF8-NEXT: [[TMP3:%.*]] = extractelement <8 x i1> [[TMP1]], i64 0 |
| ; VF8-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] |
| ; VF8: [[PRED_STORE_IF]]: |
| ; VF8-NEXT: [[TMP4:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 0 |
| ; VF8-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP4]] |
| ; VF8-NEXT: [[TMP6:%.*]] = extractelement <8 x i64> [[TMP2]], i64 0 |
| ; VF8-NEXT: store i64 [[TMP6]], ptr [[TMP5]], align 8 |
| ; VF8-NEXT: br label %[[PRED_STORE_CONTINUE]] |
| ; VF8: [[PRED_STORE_CONTINUE]]: |
| ; VF8-NEXT: [[TMP7:%.*]] = extractelement <8 x i1> [[TMP1]], i64 1 |
| ; VF8-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]] |
| ; VF8: [[PRED_STORE_IF1]]: |
| ; VF8-NEXT: [[TMP8:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 1 |
| ; VF8-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP8]] |
| ; VF8-NEXT: [[TMP10:%.*]] = extractelement <8 x i64> [[TMP2]], i64 1 |
| ; VF8-NEXT: store i64 [[TMP10]], ptr [[TMP9]], align 8 |
| ; VF8-NEXT: br label %[[PRED_STORE_CONTINUE2]] |
| ; VF8: [[PRED_STORE_CONTINUE2]]: |
| ; VF8-NEXT: [[TMP11:%.*]] = extractelement <8 x i1> [[TMP1]], i64 2 |
| ; VF8-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]] |
| ; VF8: [[PRED_STORE_IF3]]: |
| ; VF8-NEXT: [[TMP12:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 2 |
| ; VF8-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP12]] |
| ; VF8-NEXT: [[TMP14:%.*]] = extractelement <8 x i64> [[TMP2]], i64 2 |
| ; VF8-NEXT: store i64 [[TMP14]], ptr [[TMP13]], align 8 |
| ; VF8-NEXT: br label %[[PRED_STORE_CONTINUE4]] |
| ; VF8: [[PRED_STORE_CONTINUE4]]: |
| ; VF8-NEXT: [[TMP15:%.*]] = extractelement <8 x i1> [[TMP1]], i64 3 |
| ; VF8-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]] |
| ; VF8: [[PRED_STORE_IF5]]: |
| ; VF8-NEXT: [[TMP16:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 3 |
| ; VF8-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP16]] |
| ; VF8-NEXT: [[TMP18:%.*]] = extractelement <8 x i64> [[TMP2]], i64 3 |
| ; VF8-NEXT: store i64 [[TMP18]], ptr [[TMP17]], align 8 |
| ; VF8-NEXT: br label %[[PRED_STORE_CONTINUE6]] |
| ; VF8: [[PRED_STORE_CONTINUE6]]: |
| ; VF8-NEXT: [[TMP19:%.*]] = extractelement <8 x i1> [[TMP1]], i64 4 |
| ; VF8-NEXT: br i1 [[TMP19]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]] |
| ; VF8: [[PRED_STORE_IF7]]: |
| ; VF8-NEXT: [[TMP20:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 4 |
| ; VF8-NEXT: [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP20]] |
| ; VF8-NEXT: [[TMP22:%.*]] = extractelement <8 x i64> [[TMP2]], i64 4 |
| ; VF8-NEXT: store i64 [[TMP22]], ptr [[TMP21]], align 8 |
| ; VF8-NEXT: br label %[[PRED_STORE_CONTINUE8]] |
| ; VF8: [[PRED_STORE_CONTINUE8]]: |
| ; VF8-NEXT: [[TMP23:%.*]] = extractelement <8 x i1> [[TMP1]], i64 5 |
| ; VF8-NEXT: br i1 [[TMP23]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]] |
| ; VF8: [[PRED_STORE_IF9]]: |
| ; VF8-NEXT: [[TMP24:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 5 |
| ; VF8-NEXT: [[TMP25:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP24]] |
| ; VF8-NEXT: [[TMP26:%.*]] = extractelement <8 x i64> [[TMP2]], i64 5 |
| ; VF8-NEXT: store i64 [[TMP26]], ptr [[TMP25]], align 8 |
| ; VF8-NEXT: br label %[[PRED_STORE_CONTINUE10]] |
| ; VF8: [[PRED_STORE_CONTINUE10]]: |
| ; VF8-NEXT: [[TMP27:%.*]] = extractelement <8 x i1> [[TMP1]], i64 6 |
| ; VF8-NEXT: br i1 [[TMP27]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]] |
| ; VF8: [[PRED_STORE_IF11]]: |
| ; VF8-NEXT: [[TMP28:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 6 |
| ; VF8-NEXT: [[TMP29:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP28]] |
| ; VF8-NEXT: [[TMP30:%.*]] = extractelement <8 x i64> [[TMP2]], i64 6 |
| ; VF8-NEXT: store i64 [[TMP30]], ptr [[TMP29]], align 8 |
| ; VF8-NEXT: br label %[[PRED_STORE_CONTINUE12]] |
| ; VF8: [[PRED_STORE_CONTINUE12]]: |
| ; VF8-NEXT: [[TMP31:%.*]] = extractelement <8 x i1> [[TMP1]], i64 7 |
| ; VF8-NEXT: br i1 [[TMP31]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14]] |
| ; VF8: [[PRED_STORE_IF13]]: |
| ; VF8-NEXT: [[TMP32:%.*]] = extractelement <8 x i64> [[WIDE_LOAD]], i64 7 |
| ; VF8-NEXT: [[TMP33:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[TMP32]] |
| ; VF8-NEXT: [[TMP34:%.*]] = extractelement <8 x i64> [[TMP2]], i64 7 |
| ; VF8-NEXT: store i64 [[TMP34]], ptr [[TMP33]], align 8 |
| ; VF8-NEXT: br label %[[PRED_STORE_CONTINUE14]] |
| ; VF8: [[PRED_STORE_CONTINUE14]]: |
| ; VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 |
| ; VF8-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; VF8-NEXT: br i1 [[TMP35]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; VF8: [[MIDDLE_BLOCK]]: |
| ; VF8-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; VF8-NEXT: br i1 [[CMP_N]], label %[[END:.*]], label %[[SCALAR_PH]] |
| ; VF8: [[SCALAR_PH]]: |
| ; VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; VF8-NEXT: br label %[[FOR_BODY:.*]] |
| ; VF8: [[FOR_BODY]]: |
| ; VF8-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_LOOP:.*]] ] |
| ; VF8-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]] |
| ; VF8-NEXT: [[IDX:%.*]] = load i64, ptr [[LD_ADDR]], align 8 |
| ; VF8-NEXT: [[IFCOND:%.*]] = icmp ult i64 [[IDX]], 5 |
| ; VF8-NEXT: br i1 [[IFCOND]], label %[[IF_THEN:.*]], label %[[FOR_LOOP]] |
| ; VF8: [[IF_THEN]]: |
| ; VF8-NEXT: [[FOO_RET:%.*]] = call i64 @foo(i64 [[IDX]]) #[[ATTR0:[0-9]+]] |
| ; VF8-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds i64, ptr [[DEST]], i64 [[IDX]] |
| ; VF8-NEXT: store i64 [[FOO_RET]], ptr [[ST_ADDR]], align 8 |
| ; VF8-NEXT: br label %[[FOR_LOOP]] |
| ; VF8: [[FOR_LOOP]]: |
| ; VF8-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 |
| ; VF8-NEXT: [[LOOPCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] |
| ; VF8-NEXT: br i1 [[LOOPCOND]], label %[[END]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] |
| ; VF8: [[END]]: |
| ; VF8-NEXT: ret void |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.loop ] |
| %ld.addr = getelementptr inbounds i64, ptr %src, i64 %iv |
| %idx = load i64, ptr %ld.addr, align 8 |
| %ifcond = icmp ult i64 %idx, 5 |
| br i1 %ifcond, label %if.then, label %for.loop |
| |
| if.then: |
| %foo.ret = call i64 @foo(i64 %idx) #0 |
| ; A scatter to a loaded index: not consecutive, so the store is |
| ; scalar-with-predication and triggers the predication-discount analysis. |
| %st.addr = getelementptr inbounds i64, ptr %dest, i64 %idx |
| store i64 %foo.ret, ptr %st.addr, align 8 |
| br label %for.loop |
| |
| for.loop: |
| %iv.next = add nsw nuw i64 %iv, 1 |
| %loopcond = icmp eq i64 %iv.next, %N |
| br i1 %loopcond, label %end, label %for.body |
| |
| end: |
| ret void |
| } |
| |
| declare i64 @foo(i64) #0 |
| declare <2 x i64> @vector_foo_2(<2 x i64>, <2 x i1>) |
| declare <8 x i64> @vector_foo_8(<8 x i64>, <8 x i1>) |
| |
| ; Masked vector variants for VF=2 and VF=8, so the call is a widen-with-mask |
| ; candidate at both VFs while feeding the scalar-with-predication scatter store. |
| attributes #0 = { readonly nounwind "vector-function-abi-variant"="_ZGV_LLVM_M2v_foo(vector_foo_2),_ZGV_LLVM_M8v_foo(vector_foo_8)" } |
