llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv-btc.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6

; This file complements vplan-based-stride-mv.ll by containing the test
; functions from VPlan/vplan-based-stride-mv.ll that focus on interactions
; between stride speculation and backedge taken count as we want to look/verify
; both fixed and scalable vector width vectorization.

; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
; RUN:     -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops \
; RUN:     -enable-mem-access-versioning=false | FileCheck %s --check-prefix COMPARE-NO-MV

; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
; RUN:     -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops \
; RUN:     -enable-mem-access-versioning=true | FileCheck %s --check-prefix COMPARE-LAA-MV

; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
; RUN:     -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops \
; RUN:     -tail-folding-policy=must-fold-tail \
; RUN:     -enable-mem-access-versioning=false | FileCheck %s --check-prefix COMPARE-NO-MV-FOLD-TAIL

; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
; RUN:     -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops \
; RUN:     -tail-folding-policy=must-fold-tail \
; RUN:     -enable-mem-access-versioning=true | FileCheck %s --check-prefix COMPARE-LAA-MV-FOLD_TAIL

; RUN: opt < %s -p loop-vectorize -force-vector-width="vscale x 4" -S \
; RUN:     -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops  -force-target-supports-scalable-vectors \
; RUN:     -enable-mem-access-versioning=false | FileCheck %s --check-prefix COMPARE-NO-MV-SCALABLE

; RUN: opt < %s -p loop-vectorize -force-vector-width="vscale x 4" -S \
; RUN:     -force-target-supports-masked-memory-ops -force-target-supports-gather-scatter-ops  -force-target-supports-scalable-vectors \
; RUN:     -enable-mem-access-versioning=true | FileCheck %s --check-prefix COMPARE-LAA-MV-SCALABLE

; Not valuable by itself, but as a basis for the subsequent test to ensure that
; non-constant trip count doesn't change anything by itself. Also show the order
; of checks between stride speculation and trip-count check when not
; tail-folding.
define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @non_constant_btc(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; COMPARE-NO-MV-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV:       [[VECTOR_PH]]:
; COMPARE-NO-MV-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT:    [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT:    [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
; COMPARE-NO-MV-NEXT:    [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-NO-MV-NEXT:    [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT:    store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
; COMPARE-NO-MV-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT:    [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; COMPARE-NO-MV:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV:       [[SCALAR_PH]]:
; COMPARE-NO-MV-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; COMPARE-NO-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV:       [[HEADER]]:
; COMPARE-NO-MV-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; COMPARE-NO-MV:       [[EXIT]]:
; COMPARE-NO-MV-NEXT:    ret void
;
; COMPARE-LAA-MV-LABEL: define void @non_constant_btc(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; COMPARE-LAA-MV-NEXT:  [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-LAA-MV-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-LAA-MV-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-LAA-MV-NEXT:    [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-LAA-MV-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV:       [[VECTOR_BODY]]:
; COMPARE-LAA-MV-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-NEXT:    [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
; COMPARE-LAA-MV-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
; COMPARE-LAA-MV-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-LAA-MV-NEXT:    store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; COMPARE-LAA-MV:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-LAA-MV:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; COMPARE-LAA-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV:       [[HEADER]]:
; COMPARE-LAA-MV-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; COMPARE-LAA-MV:       [[EXIT]]:
; COMPARE-LAA-MV-NEXT:    ret void
;
; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @non_constant_btc(
; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; COMPARE-NO-MV-FOLD-TAIL-NEXT:  [[ENTRY:.*:]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[VECTOR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP27:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP1]], <4 x i64> poison)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP28:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP27]], ptr align 8 [[TMP28]], <4 x i1> [[TMP1]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[EXIT:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[EXIT]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    ret void
;
; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @non_constant_btc(
; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:  [[ENTRY:.*:]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_BODY]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP17:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP18]], <4 x i1> [[TMP1]], <4 x i64> poison)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP17]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[TMP20]], <4 x i1> [[TMP1]])
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[INDEX_NEXT]] = add i64 [[TMP17]], 4
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[EXIT:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[HEADER]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[EXIT]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    ret void
;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @non_constant_btc(
; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; COMPARE-NO-MV-SCALABLE-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-SCALABLE:       [[VECTOR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-SCALABLE:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    store <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP6]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; COMPARE-NO-MV-SCALABLE:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV-SCALABLE:       [[SCALAR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV-SCALABLE:       [[HEADER]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-SCALABLE-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; COMPARE-NO-MV-SCALABLE:       [[EXIT]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @non_constant_btc(
; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; COMPARE-LAA-MV-SCALABLE-NEXT:  [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[VECTOR_BODY]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP4]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    store <vscale x 4 x i64> [[WIDE_LOAD]], ptr [[TMP5]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; COMPARE-LAA-MV-SCALABLE:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-LAA-MV-SCALABLE:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[HEADER]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; COMPARE-LAA-MV-SCALABLE:       [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    ret void
;
entry:
  br label %header

header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
  %iv.next = add nsw i64 %iv, 1

  %idx = mul i64 %iv, %stride

  %gep.ld = getelementptr i64, ptr %p, i64 %idx
  %ld = load i64, ptr %gep.ld, align 8

  %gep.st = getelementptr i64, ptr %p.out, i64 %iv
  store i64 %ld, ptr %gep.st, align 8

  %exitcond = icmp slt i64 %iv.next, %n
  br i1 %exitcond, label %header, label %exit

exit:
  ret void
}

; BTC == stride, so stride speculation would result in zero vector loop
; iterations.
define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @stride_as_btc(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
; COMPARE-NO-MV-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV:       [[VECTOR_PH]]:
; COMPARE-NO-MV-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT:    [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT:    [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
; COMPARE-NO-MV-NEXT:    [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-NO-MV-NEXT:    [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT:    store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
; COMPARE-NO-MV-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT:    [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; COMPARE-NO-MV:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV:       [[SCALAR_PH]]:
; COMPARE-NO-MV-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; COMPARE-NO-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV:       [[HEADER]]:
; COMPARE-NO-MV-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
; COMPARE-NO-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; COMPARE-NO-MV:       [[EXIT]]:
; COMPARE-NO-MV-NEXT:    ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_as_btc(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-NEXT:  [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV:       [[HEADER]]:
; COMPARE-LAA-MV-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV:       [[EXIT]]:
; COMPARE-LAA-MV-NEXT:    ret void
;
; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @stride_as_btc(
; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-FOLD-TAIL-NEXT:  [[ENTRY:.*:]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[VECTOR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP27:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP1]], <4 x i64> poison)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP28:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP27]], ptr align 8 [[TMP28]], <4 x i1> [[TMP1]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[EXIT:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[EXIT]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    ret void
;
; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_as_btc(
; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:  [[ENTRY:.*:]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[PRED_STORE_CONTINUE6:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[PRED_STORE_CONTINUE6]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[P]], <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x i64> poison)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[P_OUT]], <4 x i1> <i1 true, i1 false, i1 false, i1 false>)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[EXIT:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[HEADER]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP4:![0-9]+]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[EXIT]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    ret void
;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_as_btc(
; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-SCALABLE-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-SCALABLE:       [[VECTOR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-SCALABLE:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    store <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP6]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; COMPARE-NO-MV-SCALABLE:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV-SCALABLE:       [[SCALAR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV-SCALABLE:       [[HEADER]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-SCALABLE-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; COMPARE-NO-MV-SCALABLE:       [[EXIT]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_as_btc(
; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-SCALABLE-NEXT:  [[SCALAR_PH:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[HEADER]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    ret void
;
entry:
  br label %header

header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
  %iv.next = add nsw i64 %iv, 1

  %idx = mul i64 %iv, %stride

  %gep.ld = getelementptr i64, ptr %p, i64 %idx
  %ld = load i64, ptr %gep.ld, align 8

  %gep.st = getelementptr i64, ptr %p.out, i64 %iv
  store i64 %ld, ptr %gep.st, align 8

  %exitcond = icmp slt i64 %iv.next, %stride
  br i1 %exitcond, label %header, label %exit

exit:
  ret void
}

; Similar to above but a slightly more complex dependency between stride and
; BTC.
define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @stride_dependent_btc(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV:       [[VECTOR_PH]]:
; COMPARE-NO-MV-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT:    [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT:    [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
; COMPARE-NO-MV-NEXT:    [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-NO-MV-NEXT:    [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT:    store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
; COMPARE-NO-MV-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT:    [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; COMPARE-NO-MV:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV:       [[SCALAR_PH]]:
; COMPARE-NO-MV-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; COMPARE-NO-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV:       [[HEADER]]:
; COMPARE-NO-MV-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; COMPARE-NO-MV:       [[EXIT]]:
; COMPARE-NO-MV-NEXT:    ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_dependent_btc(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-NEXT:  [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV:       [[HEADER]]:
; COMPARE-LAA-MV-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV:       [[EXIT]]:
; COMPARE-LAA-MV-NEXT:    ret void
;
; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @stride_dependent_btc(
; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-FOLD-TAIL-NEXT:  [[ENTRY:.*:]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[VECTOR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP27:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP1]], <4 x i64> poison)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP28:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP27]], ptr align 8 [[TMP28]], <4 x i1> [[TMP1]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[EXIT:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[EXIT]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    ret void
;
; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_dependent_btc(
; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:  [[ENTRY:.*:]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[PRED_STORE_CONTINUE6:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[PRED_STORE_CONTINUE6]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[P]], <4 x i1> <i1 true, i1 true, i1 false, i1 false>, <4 x i64> poison)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[P_OUT]], <4 x i1> <i1 true, i1 true, i1 false, i1 false>)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[EXIT:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[HEADER]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[EXIT]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    ret void
;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_dependent_btc(
; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-SCALABLE-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 1
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-SCALABLE:       [[VECTOR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-SCALABLE:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    store <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP6]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; COMPARE-NO-MV-SCALABLE:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV-SCALABLE:       [[SCALAR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV-SCALABLE:       [[HEADER]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-SCALABLE-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; COMPARE-NO-MV-SCALABLE:       [[EXIT]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_dependent_btc(
; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-SCALABLE-NEXT:  [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[HEADER]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    ret void
;
entry:
  %n = add i64 %stride, 1
  br label %header

header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
  %iv.next = add nsw i64 %iv, 1

  %idx = mul i64 %iv, %stride

  %gep.ld = getelementptr i64, ptr %p, i64 %idx
  %ld = load i64, ptr %gep.ld, align 8

  %gep.st = getelementptr i64, ptr %p.out, i64 %iv
  store i64 %ld, ptr %gep.st, align 8

  %exitcond = icmp slt i64 %iv.next, %n
  br i1 %exitcond, label %header, label %exit

exit:
  ret void
}

; BTC dependent on %stride, but stride speculation doesn't necessarily mean no
; vector loop iterations. The test shows in which order we emit the run time
; checks for both.
define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i64 %m) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @stride_btc_checks_order(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT:    [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
; COMPARE-NO-MV-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV:       [[VECTOR_PH]]:
; COMPARE-NO-MV-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT:    [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT:    [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
; COMPARE-NO-MV-NEXT:    [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-NO-MV-NEXT:    [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT:    store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
; COMPARE-NO-MV-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT:    [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; COMPARE-NO-MV:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV:       [[SCALAR_PH]]:
; COMPARE-NO-MV-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; COMPARE-NO-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV:       [[HEADER]]:
; COMPARE-NO-MV-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
; COMPARE-NO-MV:       [[EXIT]]:
; COMPARE-NO-MV-NEXT:    ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_btc_checks_order(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-NEXT:  [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT:    [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT:    [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
; COMPARE-LAA-MV-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX1]], 4
; COMPARE-LAA-MV-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[SMAX1]], 3
; COMPARE-LAA-MV-NEXT:    [[N_VEC:%.*]] = sub i64 [[SMAX1]], [[N_MOD_VF]]
; COMPARE-LAA-MV-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV:       [[VECTOR_BODY]]:
; COMPARE-LAA-MV-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-NEXT:    [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
; COMPARE-LAA-MV-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
; COMPARE-LAA-MV-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-LAA-MV-NEXT:    store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; COMPARE-LAA-MV:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX1]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-LAA-MV:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; COMPARE-LAA-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV:       [[HEADER]]:
; COMPARE-LAA-MV-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; COMPARE-LAA-MV:       [[EXIT]]:
; COMPARE-LAA-MV-NEXT:    ret void
;
; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @stride_btc_checks_order(
; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-FOLD-TAIL-NEXT:  [[ENTRY:.*:]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[VECTOR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP27:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP1]], <4 x i64> poison)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP28:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP27]], ptr align 8 [[TMP28]], <4 x i1> [[TMP1]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[EXIT:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[EXIT]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    ret void
;
; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_btc_checks_order(
; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:  [[ENTRY:.*:]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_BODY]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP17:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP18]], <4 x i1> [[TMP1]], <4 x i64> poison)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP17]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[TMP20]], <4 x i1> [[TMP1]])
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[INDEX_NEXT]] = add i64 [[TMP17]], 4
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[EXIT:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[HEADER]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[EXIT]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    ret void
;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_btc_checks_order(
; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-SCALABLE-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-SCALABLE:       [[VECTOR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-SCALABLE:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    store <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP6]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; COMPARE-NO-MV-SCALABLE:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV-SCALABLE:       [[SCALAR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV-SCALABLE:       [[HEADER]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-SCALABLE-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
; COMPARE-NO-MV-SCALABLE:       [[EXIT]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_btc_checks_order(
; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-SCALABLE-NEXT:  [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[VECTOR_BODY]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP4]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    store <vscale x 4 x i64> [[WIDE_LOAD]], ptr [[TMP5]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; COMPARE-LAA-MV-SCALABLE:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-LAA-MV-SCALABLE:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[HEADER]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; COMPARE-LAA-MV-SCALABLE:       [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    ret void
;
entry:
  %n = mul i64 %m, %stride
  br label %header

header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
  %iv.next = add nsw i64 %iv, 1

  %idx = mul i64 %iv, %stride

  %gep.ld = getelementptr i64, ptr %p, i64 %idx
  %ld = load i64, ptr %gep.ld, align 8

  %gep.st = getelementptr i64, ptr %p.out, i64 %iv
  store i64 %ld, ptr %gep.st, align 8

  %exitcond = icmp slt i64 %iv.next, %n
  br i1 %exitcond, label %header, label %exit

exit:
  ret void
}

; BTC fully defined by stride speculation but still allows vector loop execution.
define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64 %stride) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @stride_dependent_btc_non_preventive(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-NO-MV-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV:       [[VECTOR_PH]]:
; COMPARE-NO-MV-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT:    [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT:    [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
; COMPARE-NO-MV-NEXT:    [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-NO-MV-NEXT:    [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT:    store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
; COMPARE-NO-MV-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT:    [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; COMPARE-NO-MV:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV:       [[SCALAR_PH]]:
; COMPARE-NO-MV-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; COMPARE-NO-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV:       [[HEADER]]:
; COMPARE-NO-MV-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
; COMPARE-NO-MV:       [[EXIT]]:
; COMPARE-NO-MV-NEXT:    ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_dependent_btc_non_preventive(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-NEXT:  [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-LAA-MV-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV:       [[VECTOR_BODY]]:
; COMPARE-LAA-MV-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
; COMPARE-LAA-MV-NEXT:    store <4 x i64> [[WIDE_LOAD]], ptr [[P_OUT]], align 8
; COMPARE-LAA-MV-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-LAA-MV:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT:    br label %[[EXIT:.*]]
; COMPARE-LAA-MV:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV:       [[HEADER]]:
; COMPARE-LAA-MV-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP6:![0-9]+]]
; COMPARE-LAA-MV:       [[EXIT]]:
; COMPARE-LAA-MV-NEXT:    ret void
;
; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @stride_dependent_btc_non_preventive(
; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-FOLD-TAIL-NEXT:  [[ENTRY:.*:]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[VECTOR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP2:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP2]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP27:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP1]], <4 x i64> poison)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP28:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP27]], ptr align 8 [[TMP28]], <4 x i1> [[TMP1]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[EXIT:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[EXIT]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    ret void
;
; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_dependent_btc_non_preventive(
; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:  [[ENTRY:.*:]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[PRED_STORE_CONTINUE6:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[PRED_STORE_CONTINUE6]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[P]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[P_OUT]], <4 x i1> splat (i1 true))
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[EXIT:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[HEADER]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[EXIT]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    ret void
;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_dependent_btc_non_preventive(
; COMPARE-NO-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-SCALABLE-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-SCALABLE:       [[VECTOR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-SCALABLE:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP5:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP5]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP6:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    store <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], ptr [[TMP6]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; COMPARE-NO-MV-SCALABLE:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV-SCALABLE:       [[SCALAR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV-SCALABLE:       [[HEADER]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-SCALABLE-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
; COMPARE-NO-MV-SCALABLE:       [[EXIT]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_dependent_btc_non_preventive(
; COMPARE-LAA-MV-SCALABLE-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-SCALABLE-NEXT:  [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-LAA-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[HEADER]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    ret void
;
entry:
  %n = add i64 %stride, 3
  br label %header

header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
  %iv.next = add nsw i64 %iv, 1

  %idx = mul i64 %iv, %stride

  %gep.ld = getelementptr i64, ptr %p, i64 %idx
  %ld = load i64, ptr %gep.ld, align 8

  %gep.st = getelementptr i64, ptr %p.out, i64 %iv
  store i64 %ld, ptr %gep.st, align 8

  %exitcond = icmp slt i64 %iv.next, %n
  br i1 %exitcond, label %header, label %exit

exit:
  ret void
}

; Doesn't pass legality as run-time memory dependencies check doesn't support
; strided accesses. If it did, the purpose of this test would be to show how all
; three run-time checks (mem deps/stride speculation/trip-count) would be
; ordered in respect to each other. Note that pointer aliasing check could
; potentially be simplified if done after stride speculation. However, that
; isn't necessarily the best idea because we could also multi-version for stride
; and keep aliasing part generic and shared by both vector loops..
define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @stride_btc_memdep_triple_check(
; COMPARE-NO-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT:    [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-NO-MV-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-NO-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV:       [[HEADER]]:
; COMPARE-NO-MV-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-NO-MV:       [[EXIT]]:
; COMPARE-NO-MV-NEXT:    ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_btc_memdep_triple_check(
; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-NEXT:  [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT:    [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-LAA-MV-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-LAA-MV:       [[VECTOR_MEMCHECK]]:
; COMPARE-LAA-MV-NEXT:    [[TMP0:%.*]] = sub i64 [[OUT_OFFSET]], 1
; COMPARE-LAA-MV-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
; COMPARE-LAA-MV-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV:       [[VECTOR_BODY]]:
; COMPARE-LAA-MV-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
; COMPARE-LAA-MV-NEXT:    store <4 x i64> [[WIDE_LOAD]], ptr [[P_OUT]], align 8
; COMPARE-LAA-MV-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-LAA-MV:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT:    br label %[[EXIT:.*]]
; COMPARE-LAA-MV:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; COMPARE-LAA-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV:       [[HEADER]]:
; COMPARE-LAA-MV-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; COMPARE-LAA-MV:       [[EXIT]]:
; COMPARE-LAA-MV-NEXT:    ret void
;
; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @stride_btc_memdep_triple_check(
; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-FOLD-TAIL-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[HEADER]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[EXIT]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    ret void
;
; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_btc_memdep_triple_check(
; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:  [[ENTRY:.*:]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_MEMCHECK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP0:%.*]] = sub i64 [[OUT_OFFSET]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[PRED_STORE_CONTINUE6:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[PRED_STORE_CONTINUE6]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[P]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[WIDE_MASKED_LOAD]], ptr align 8 [[P_OUT]], <4 x i1> splat (i1 true))
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[EXIT:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[HEADER]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[EXIT]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    ret void
;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_btc_memdep_triple_check(
; COMPARE-NO-MV-SCALABLE-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-SCALABLE-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-NO-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV-SCALABLE:       [[HEADER]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-SCALABLE-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-NO-MV-SCALABLE:       [[EXIT]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_btc_memdep_triple_check(
; COMPARE-LAA-MV-SCALABLE-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-SCALABLE-NEXT:  [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-LAA-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[HEADER]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    ret void
;
entry:
  %p.out = getelementptr i8, ptr %p, i64 %out.offset
  %n = add i64 %stride, 3
  br label %header

header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
  %iv.next = add nsw i64 %iv, 1

  %idx = mul i64 %iv, %stride

  %gep.ld = getelementptr i64, ptr %p, i64 %idx
  %ld = load i64, ptr %gep.ld, align 8

  %gep.st = getelementptr i64, ptr %p.out, i64 %iv
  store i64 %ld, ptr %gep.st, align 8

  %exitcond = icmp slt i64 %iv.next, %n
  br i1 %exitcond, label %header, label %exit

exit:
  ret void
}

; Same as above but memdep check doesn't depend on stride
define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2, i64 %stride, i64 %out.offset) vscale_range(2, 1024) {
; COMPARE-NO-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
; COMPARE-NO-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT:    [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-NO-MV-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-NO-MV-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-NO-MV:       [[VECTOR_MEMCHECK]]:
; COMPARE-NO-MV-NEXT:    [[TMP21:%.*]] = sub i64 [[OUT_OFFSET]], 1
; COMPARE-NO-MV-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], 31
; COMPARE-NO-MV-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV:       [[VECTOR_PH]]:
; COMPARE-NO-MV-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 3
; COMPARE-NO-MV-NEXT:    [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
; COMPARE-NO-MV-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT:    [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
; COMPARE-NO-MV-NEXT:    [[TMP16:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-NO-MV-NEXT:    [[TMP17:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP17]], align 8
; COMPARE-NO-MV-NEXT:    [[TMP18:%.*]] = add <4 x i64> [[TMP16]], [[WIDE_LOAD]]
; COMPARE-NO-MV-NEXT:    [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT:    store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
; COMPARE-NO-MV-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT:    [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; COMPARE-NO-MV:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV:       [[SCALAR_PH]]:
; COMPARE-NO-MV-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; COMPARE-NO-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV:       [[HEADER]]:
; COMPARE-NO-MV-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-NEXT:    [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
; COMPARE-NO-MV-NEXT:    [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
; COMPARE-NO-MV-NEXT:    [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
; COMPARE-NO-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT:    store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
; COMPARE-NO-MV:       [[EXIT]]:
; COMPARE-NO-MV-NEXT:    ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-NEXT:  [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT:    [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-LAA-MV-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-LAA-MV:       [[VECTOR_MEMCHECK]]:
; COMPARE-LAA-MV-NEXT:    [[TMP1:%.*]] = sub i64 [[OUT_OFFSET]], 1
; COMPARE-LAA-MV-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 31
; COMPARE-LAA-MV-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV:       [[VECTOR_BODY]]:
; COMPARE-LAA-MV-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
; COMPARE-LAA-MV-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[P2]], align 8
; COMPARE-LAA-MV-NEXT:    [[TMP0:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; COMPARE-LAA-MV-NEXT:    store <4 x i64> [[TMP0]], ptr [[P_OUT]], align 8
; COMPARE-LAA-MV-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-LAA-MV:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT:    br label %[[EXIT:.*]]
; COMPARE-LAA-MV:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; COMPARE-LAA-MV-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV:       [[HEADER]]:
; COMPARE-LAA-MV-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-NEXT:    [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT:    [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
; COMPARE-LAA-MV-NEXT:    [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
; COMPARE-LAA-MV-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT:    store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
; COMPARE-LAA-MV:       [[EXIT]]:
; COMPARE-LAA-MV-NEXT:    ret void
;
; COMPARE-NO-MV-FOLD-TAIL-LABEL: define void @stride_btc_independent_memdep_triple_check(
; COMPARE-NO-MV-FOLD-TAIL-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-FOLD-TAIL-NEXT:  [[ENTRY:.*:]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[VECTOR_MEMCHECK]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP1:%.*]] = sub i64 [[OUT_OFFSET]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 31
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[VECTOR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP6]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP45:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> [[TMP2]], <4 x i64> poison)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP30:%.*]] = getelementptr i64, ptr [[P2]], i64 [[TMP5]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP46:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP30]], <4 x i1> [[TMP2]], <4 x i64> poison)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP47:%.*]] = add <4 x i64> [[TMP45]], [[TMP46]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP54:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP5]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP47]], ptr align 8 [[TMP54]], <4 x i1> [[TMP2]])
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[INDEX_NEXT]] = add i64 [[TMP5]], 4
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[TMP56:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br i1 [[TMP56]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[EXIT:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[SCALAR_PH]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV-FOLD-TAIL:       [[HEADER]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
; COMPARE-NO-MV-FOLD-TAIL:       [[EXIT]]:
; COMPARE-NO-MV-FOLD-TAIL-NEXT:    ret void
;
; COMPARE-LAA-MV-FOLD_TAIL-LABEL: define void @stride_btc_independent_memdep_triple_check(
; COMPARE-LAA-MV-FOLD_TAIL-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:  [[ENTRY:.*:]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_MEMCHECK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP0:%.*]] = sub i64 [[OUT_OFFSET]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[VECTOR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[PRED_LOAD_CONTINUE6:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[PRED_LOAD_CONTINUE6]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP29:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[P]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP30:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[P2]], <4 x i1> splat (i1 true), <4 x i64> poison)
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[TMP31:%.*]] = add <4 x i64> [[TMP29]], [[TMP30]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP31]], ptr align 8 [[P_OUT]], <4 x i1> splat (i1 true))
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[EXIT:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[SCALAR_PH]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[HEADER]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP10:![0-9]+]]
; COMPARE-LAA-MV-FOLD_TAIL:       [[EXIT]]:
; COMPARE-LAA-MV-FOLD_TAIL-NEXT:    ret void
;
; COMPARE-NO-MV-SCALABLE-LABEL: define void @stride_btc_independent_memdep_triple_check(
; COMPARE-NO-MV-SCALABLE-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-NO-MV-SCALABLE-NEXT:  [[ENTRY:.*]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; COMPARE-NO-MV-SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-NO-MV-SCALABLE:       [[VECTOR_MEMCHECK]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 5
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP6:%.*]] = add nsw i64 [[TMP4]], -1
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP7:%.*]] = sub i64 [[OUT_OFFSET]], 1
; COMPARE-NO-MV-SCALABLE-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP7]], [[TMP6]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV-SCALABLE:       [[VECTOR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; COMPARE-NO-MV-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV-SCALABLE:       [[VECTOR_BODY]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP9]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <vscale x 4 x i64> [[TMP10]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i64> poison)
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP11:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP11]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], [[WIDE_LOAD]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    store <vscale x 4 x i64> [[TMP12]], ptr [[TMP13]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; COMPARE-NO-MV-SCALABLE:       [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV-SCALABLE:       [[SCALAR_PH]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-NO-MV-SCALABLE:       [[HEADER]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
; COMPARE-NO-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-SCALABLE-NEXT:    store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-NO-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
; COMPARE-NO-MV-SCALABLE:       [[EXIT]]:
; COMPARE-NO-MV-SCALABLE-NEXT:    ret void
;
; COMPARE-LAA-MV-SCALABLE-LABEL: define void @stride_btc_independent_memdep_triple_check(
; COMPARE-LAA-MV-SCALABLE-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) #[[ATTR0]] {
; COMPARE-LAA-MV-SCALABLE-NEXT:  [[ENTRY:.*]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[N:%.*]] = add i64 [[STRIDE]], 3
; COMPARE-LAA-MV-SCALABLE-NEXT:    br label %[[HEADER:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[HEADER]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_LD2:%.*]] = getelementptr i64, ptr [[P2]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[LD2:%.*]] = load i64, ptr [[GEP_LD2]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[VAL:%.*]] = add i64 [[LD]], [[LD2]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-SCALABLE-NEXT:    [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
; COMPARE-LAA-MV-SCALABLE-NEXT:    br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV-SCALABLE:       [[EXIT]]:
; COMPARE-LAA-MV-SCALABLE-NEXT:    ret void
;
entry:
  %p.out = getelementptr i8, ptr %p2, i64 %out.offset
  %n = add i64 %stride, 3
  br label %header

header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %header ]
  %iv.next = add nsw i64 %iv, 1

  %idx = mul i64 %iv, %stride

  %gep.ld = getelementptr i64, ptr %p, i64 %idx
  %ld = load i64, ptr %gep.ld, align 8

  %gep.ld2 = getelementptr i64, ptr %p2, i64 %iv
  %ld2 = load i64, ptr %gep.ld2, align 8

  %val = add i64 %ld, %ld2

  %gep.st = getelementptr i64, ptr %p.out, i64 %iv
  store i64 %val, ptr %gep.st, align 8

  %exitcond = icmp slt i64 %iv.next, %n
  br i1 %exitcond, label %header, label %exit

exit:
  ret void
}