llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 6
; RUN: opt < %s -passes=loop-vectorize -S | FileCheck %s --check-prefixes=COMMON,DEFAULT
; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-PREFER
; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=prefer-fold-tail -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-PREFER
; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-ENABLE-TP

target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target triple = "thumbv8.1m.main-arm-unknown-eabihf"

; This IR corresponds to this type of C-code:
;
;  void f(char *a, char *b, char *c, int N) {
;    while (N-- > 0)
;      *c++ = *a++ + *b++;
;  }
;
define void @sgt_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
; COMMON-LABEL: define void @sgt_loopguard(
; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; COMMON-NEXT:  [[ENTRY:.*:]]
; COMMON-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
; COMMON-NEXT:    br i1 [[CMP5]], label %[[WHILE_BODY_PREHEADER:.*]], label %[[WHILE_END:.*]]
; COMMON:       [[WHILE_BODY_PREHEADER]]:
; COMMON-NEXT:    br label %[[VECTOR_PH:.*]]
; COMMON:       [[VECTOR_PH]]:
; COMMON-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 15
; COMMON-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 15
; COMMON-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; COMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMMON:       [[VECTOR_BODY]]:
; COMMON-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMMON-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
; COMMON-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
; COMMON-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; COMMON-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
; COMMON-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT:    [[TMP0:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
; COMMON-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP0]], ptr align 1 [[NEXT_GEP]], <16 x i1> [[ACTIVE_LANE_MASK]])
; COMMON-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; COMMON-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; COMMON-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; COMMON:       [[MIDDLE_BLOCK]]:
; COMMON-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
; COMMON:       [[WHILE_END_LOOPEXIT]]:
; COMMON-NEXT:    br label %[[WHILE_END]]
; COMMON:       [[WHILE_END]]:
; COMMON-NEXT:    ret void
;
entry:
  %cmp5 = icmp sgt i32 %N, 0
  br i1 %cmp5, label %while.body.preheader, label %while.end

while.body.preheader:
  br label %while.body

while.body:
  %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
  %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]
  %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]
  %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]
  %dec = add nsw i32 %N.addr.09, -1
  %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
  %0 = load i8, ptr %a.addr.06, align 1
  %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
  %1 = load i8, ptr %b.addr.07, align 1
  %add = add i8 %1, %0
  %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
  store i8 %add, ptr %c.addr.08, align 1
  %cmp = icmp sgt i32 %N.addr.09, 1
  br i1 %cmp, label %while.body, label %while.end.loopexit

while.end.loopexit:
  br label %while.end

while.end:
  ret void
}

; No loop-guard: we need one for this to be valid.
;
define void @sgt_no_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
; COMMON-LABEL: define void @sgt_no_loopguard(
; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; COMMON-NEXT:  [[ENTRY:.*:]]
; COMMON-NEXT:    [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[N]], i32 1)
; COMMON-NEXT:    [[TMP4:%.*]] = add i32 [[N]], 1
; COMMON-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP4]], [[SMIN]]
; COMMON-NEXT:    br label %[[VECTOR_PH:.*]]
; COMMON:       [[VECTOR_PH]]:
; COMMON-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP1]], 15
; COMMON-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 15
; COMMON-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; COMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMMON:       [[VECTOR_BODY]]:
; COMMON-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMMON-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[TMP1]])
; COMMON-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
; COMMON-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; COMMON-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
; COMMON-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT:    [[TMP2:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
; COMMON-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP2]], ptr align 1 [[NEXT_GEP]], <16 x i1> [[ACTIVE_LANE_MASK]])
; COMMON-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; COMMON-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; COMMON-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; COMMON:       [[MIDDLE_BLOCK]]:
; COMMON-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
; COMMON:       [[WHILE_END_LOOPEXIT]]:
; COMMON-NEXT:    br label %[[WHILE_END:.*]]
; COMMON:       [[WHILE_END]]:
; COMMON-NEXT:    ret void
;
entry:
  br label %while.body

while.body:
  %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %entry ]
  %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %entry ]
  %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %entry ]
  %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %entry ]
  %dec = add nsw i32 %N.addr.09, -1
  %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
  %0 = load i8, ptr %a.addr.06, align 1
  %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
  %1 = load i8, ptr %b.addr.07, align 1
  %add = add i8 %1, %0
  %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
  store i8 %add, ptr %c.addr.08, align 1
  %cmp = icmp sgt i32 %N.addr.09, 1
  br i1 %cmp, label %while.body, label %while.end.loopexit

while.end.loopexit:
  br label %while.end

while.end:
  ret void
}

define void @sgt_extra_use_cmp(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
; COMMON-LABEL: define void @sgt_extra_use_cmp(
; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; COMMON-NEXT:  [[ENTRY:.*:]]
; COMMON-NEXT:    [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[N]], i32 1)
; COMMON-NEXT:    [[TMP8:%.*]] = add i32 [[N]], 1
; COMMON-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP8]], [[SMIN]]
; COMMON-NEXT:    br label %[[VECTOR_PH:.*]]
; COMMON:       [[VECTOR_PH]]:
; COMMON-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP1]], 3
; COMMON-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 3
; COMMON-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; COMMON-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N]], i64 0
; COMMON-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
; COMMON-NEXT:    [[TMP2:%.*]] = sub nsw <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
; COMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMMON:       [[VECTOR_BODY]]:
; COMMON-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMMON-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMMON-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[TMP1]])
; COMMON-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
; COMMON-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; COMMON-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
; COMMON-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[NEXT_GEP2]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison)
; COMMON-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[NEXT_GEP1]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison)
; COMMON-NEXT:    [[TMP3:%.*]] = add <4 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
; COMMON-NEXT:    [[TMP4:%.*]] = icmp sle <4 x i32> [[VEC_IND]], splat (i32 1)
; COMMON-NEXT:    [[TMP5:%.*]] = select <4 x i1> [[TMP4]], <4 x i8> [[WIDE_MASKED_LOAD3]], <4 x i8> [[WIDE_MASKED_LOAD]]
; COMMON-NEXT:    [[TMP6:%.*]] = add <4 x i8> [[TMP3]], [[TMP5]]
; COMMON-NEXT:    call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP6]], ptr align 1 [[NEXT_GEP]], <4 x i1> [[ACTIVE_LANE_MASK]])
; COMMON-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; COMMON-NEXT:    [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 -4)
; COMMON-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; COMMON-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; COMMON:       [[MIDDLE_BLOCK]]:
; COMMON-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
; COMMON:       [[WHILE_END_LOOPEXIT]]:
; COMMON-NEXT:    br label %[[WHILE_END:.*]]
; COMMON:       [[WHILE_END]]:
; COMMON-NEXT:    ret void
;
entry:
  br label %while.body

while.body:
  %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %entry ]
  %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %entry ]
  %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %entry ]
  %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %entry ]
  %dec = add nsw i32 %N.addr.09, -1
  %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
  %0 = load i8, ptr %a.addr.06, align 1
  %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
  %1 = load i8, ptr %b.addr.07, align 1
  %add = add i8 %1, %0
  %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
  %cmp = icmp sgt i32 %N.addr.09, 1
  %select = select i1 %cmp, i8 %0, i8 %1
  %add2 = add i8 %add, %select
  store i8 %add2, ptr %c.addr.08, align 1
  br i1 %cmp, label %while.body, label %while.end.loopexit

while.end.loopexit:
  br label %while.end

while.end:
  ret void
}

define void @sgt_const_tripcount(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
; COMMON-LABEL: define void @sgt_const_tripcount(
; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; COMMON-NEXT:  [[ENTRY:.*:]]
; COMMON-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
; COMMON-NEXT:    br i1 [[CMP5]], label %[[WHILE_BODY_PREHEADER:.*]], label %[[WHILE_END:.*]]
; COMMON:       [[WHILE_BODY_PREHEADER]]:
; COMMON-NEXT:    br label %[[VECTOR_PH:.*]]
; COMMON:       [[VECTOR_PH]]:
; COMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMMON:       [[VECTOR_BODY]]:
; COMMON-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMMON-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 2049)
; COMMON-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
; COMMON-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; COMMON-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
; COMMON-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT:    [[TMP0:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
; COMMON-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP0]], ptr align 1 [[NEXT_GEP]], <16 x i1> [[ACTIVE_LANE_MASK]])
; COMMON-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; COMMON-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 2064
; COMMON-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; COMMON:       [[MIDDLE_BLOCK]]:
; COMMON-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
; COMMON:       [[WHILE_END_LOOPEXIT]]:
; COMMON-NEXT:    br label %[[WHILE_END]]
; COMMON:       [[WHILE_END]]:
; COMMON-NEXT:    ret void
;
entry:
  %cmp5 = icmp sgt i32 %N, 0
  br i1 %cmp5, label %while.body.preheader, label %while.end

while.body.preheader:
  br label %while.body

while.body:
  %N.addr.09 = phi i32 [ %dec, %while.body ], [ 2049, %while.body.preheader ]
  %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]
  %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]
  %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]
  %dec = add nsw i32 %N.addr.09, -1
  %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
  %0 = load i8, ptr %a.addr.06, align 1
  %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
  %1 = load i8, ptr %b.addr.07, align 1
  %add = add i8 %1, %0
  %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
  store i8 %add, ptr %c.addr.08, align 1
  %cmp = icmp sgt i32 %N.addr.09, 1
  br i1 %cmp, label %while.body, label %while.end.loopexit

while.end.loopexit:
  br label %while.end

while.end:
  ret void
}

define void @sgt_no_guard_0_startval(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
; COMMON-LABEL: define void @sgt_no_guard_0_startval(
; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; COMMON-NEXT:  [[ENTRY:.*]]:
; COMMON-NEXT:    br label %[[WHILE_BODY:.*]]
; COMMON:       [[WHILE_BODY]]:
; COMMON-NEXT:    [[N_ADDR_09:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY]] ], [ 0, %[[ENTRY]] ]
; COMMON-NEXT:    [[C_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR4:%.*]], %[[WHILE_BODY]] ], [ [[C]], %[[ENTRY]] ]
; COMMON-NEXT:    [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], %[[WHILE_BODY]] ], [ [[B]], %[[ENTRY]] ]
; COMMON-NEXT:    [[A_ADDR_06:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[WHILE_BODY]] ], [ [[A]], %[[ENTRY]] ]
; COMMON-NEXT:    [[DEC]] = add nsw i32 [[N_ADDR_09]], -1
; COMMON-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_06]], i32 1
; COMMON-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A_ADDR_06]], align 1
; COMMON-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds i8, ptr [[B_ADDR_07]], i32 1
; COMMON-NEXT:    [[TMP1:%.*]] = load i8, ptr [[B_ADDR_07]], align 1
; COMMON-NEXT:    [[ADD:%.*]] = add i8 [[TMP1]], [[TMP0]]
; COMMON-NEXT:    [[INCDEC_PTR4]] = getelementptr inbounds i8, ptr [[C_ADDR_08]], i32 1
; COMMON-NEXT:    store i8 [[ADD]], ptr [[C_ADDR_08]], align 1
; COMMON-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N_ADDR_09]], 1
; COMMON-NEXT:    br i1 [[CMP]], label %[[WHILE_BODY]], label %[[WHILE_END_LOOPEXIT:.*]]
; COMMON:       [[WHILE_END_LOOPEXIT]]:
; COMMON-NEXT:    br label %[[WHILE_END:.*]]
; COMMON:       [[WHILE_END]]:
; COMMON-NEXT:    ret void
;
entry:
  br label %while.body

while.body:
  %N.addr.09 = phi i32 [ %dec, %while.body ], [ 0, %entry ]
  %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %entry ]
  %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %entry ]
  %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %entry]
  %dec = add nsw i32 %N.addr.09, -1
  %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
  %0 = load i8, ptr %a.addr.06, align 1
  %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
  %1 = load i8, ptr %b.addr.07, align 1
  %add = add i8 %1, %0
  %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
  store i8 %add, ptr %c.addr.08, align 1
  %cmp = icmp sgt i32 %N.addr.09, 1
  br i1 %cmp, label %while.body, label %while.end.loopexit

while.end.loopexit:
  br label %while.end

while.end:
  ret void
}

define void @sgt_step_minus_two(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
; COMMON-LABEL: define void @sgt_step_minus_two(
; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; COMMON-NEXT:  [[ENTRY:.*:]]
; COMMON-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
; COMMON-NEXT:    br i1 [[CMP5]], label %[[WHILE_BODY_PREHEADER:.*]], label %[[WHILE_END:.*]]
; COMMON:       [[WHILE_BODY_PREHEADER]]:
; COMMON-NEXT:    [[TMP0:%.*]] = lshr i32 [[N]], 1
; COMMON-NEXT:    [[TMP1:%.*]] = add nuw i32 [[TMP0]], 1
; COMMON-NEXT:    br label %[[VECTOR_PH:.*]]
; COMMON:       [[VECTOR_PH]]:
; COMMON-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP1]], 15
; COMMON-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 15
; COMMON-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; COMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMMON:       [[VECTOR_BODY]]:
; COMMON-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMMON-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[TMP1]])
; COMMON-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
; COMMON-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; COMMON-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
; COMMON-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT:    [[TMP2:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
; COMMON-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP2]], ptr align 1 [[NEXT_GEP]], <16 x i1> [[ACTIVE_LANE_MASK]])
; COMMON-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; COMMON-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; COMMON-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; COMMON:       [[MIDDLE_BLOCK]]:
; COMMON-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
; COMMON:       [[WHILE_END_LOOPEXIT]]:
; COMMON-NEXT:    br label %[[WHILE_END]]
; COMMON:       [[WHILE_END]]:
; COMMON-NEXT:    ret void
;
entry:
  %cmp5 = icmp sgt i32 %N, 0
  br i1 %cmp5, label %while.body.preheader, label %while.end

while.body.preheader:
  br label %while.body

while.body:
  %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
  %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]
  %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]
  %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]
  %dec = add nsw i32 %N.addr.09, -2
  %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
  %0 = load i8, ptr %a.addr.06, align 1
  %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
  %1 = load i8, ptr %b.addr.07, align 1
  %add = add i8 %1, %0
  %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
  store i8 %add, ptr %c.addr.08, align 1
  %cmp = icmp sgt i32 %N.addr.09, 1
  br i1 %cmp, label %while.body, label %while.end.loopexit

while.end.loopexit:
  br label %while.end

while.end:
  ret void
}

define void @sgt_step_not_constant(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N, i32 %S) #0 {
; COMMON-LABEL: define void @sgt_step_not_constant(
; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]], i32 [[S:%.*]]) #[[ATTR0]] {
; COMMON-NEXT:  [[ENTRY:.*:]]
; COMMON-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
; COMMON-NEXT:    br i1 [[CMP5]], label %[[WHILE_BODY_PREHEADER:.*]], label %[[WHILE_END:.*]]
; COMMON:       [[WHILE_BODY_PREHEADER]]:
; COMMON-NEXT:    br label %[[WHILE_BODY:.*]]
; COMMON:       [[WHILE_BODY]]:
; COMMON-NEXT:    [[N_ADDR_09:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY]] ], [ [[N]], %[[WHILE_BODY_PREHEADER]] ]
; COMMON-NEXT:    [[C_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR4:%.*]], %[[WHILE_BODY]] ], [ [[C]], %[[WHILE_BODY_PREHEADER]] ]
; COMMON-NEXT:    [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], %[[WHILE_BODY]] ], [ [[B]], %[[WHILE_BODY_PREHEADER]] ]
; COMMON-NEXT:    [[A_ADDR_06:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[WHILE_BODY]] ], [ [[A]], %[[WHILE_BODY_PREHEADER]] ]
; COMMON-NEXT:    [[DEC]] = add nsw i32 [[N_ADDR_09]], [[S]]
; COMMON-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_06]], i32 1
; COMMON-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A_ADDR_06]], align 1
; COMMON-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds i8, ptr [[B_ADDR_07]], i32 1
; COMMON-NEXT:    [[TMP1:%.*]] = load i8, ptr [[B_ADDR_07]], align 1
; COMMON-NEXT:    [[ADD:%.*]] = add i8 [[TMP1]], [[TMP0]]
; COMMON-NEXT:    [[INCDEC_PTR4]] = getelementptr inbounds i8, ptr [[C_ADDR_08]], i32 1
; COMMON-NEXT:    store i8 [[ADD]], ptr [[C_ADDR_08]], align 1
; COMMON-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N_ADDR_09]], 1
; COMMON-NEXT:    br i1 [[CMP]], label %[[WHILE_BODY]], label %[[WHILE_END_LOOPEXIT:.*]]
; COMMON:       [[WHILE_END_LOOPEXIT]]:
; COMMON-NEXT:    br label %[[WHILE_END]]
; COMMON:       [[WHILE_END]]:
; COMMON-NEXT:    ret void
;
entry:
  %cmp5 = icmp sgt i32 %N, 0
  br i1 %cmp5, label %while.body.preheader, label %while.end

while.body.preheader:
  br label %while.body

while.body:
  %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
  %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]
  %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]
  %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]
  %dec = add nsw i32 %N.addr.09, %S
  %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1
  %0 = load i8, ptr %a.addr.06, align 1
  %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1
  %1 = load i8, ptr %b.addr.07, align 1
  %add = add i8 %1, %0
  %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1
  store i8 %add, ptr %c.addr.08, align 1
  %cmp = icmp sgt i32 %N.addr.09, 1
  br i1 %cmp, label %while.body, label %while.end.loopexit

while.end.loopexit:
  br label %while.end

while.end:
  ret void
}

define void @icmp_eq(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, ptr noalias nocapture %C, i32 %N) #0 {
; COMMON-LABEL: define void @icmp_eq(
; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; COMMON-NEXT:  [[ENTRY:.*:]]
; COMMON-NEXT:    [[CMP6:%.*]] = icmp eq i32 [[N]], 0
; COMMON-NEXT:    br i1 [[CMP6]], label %[[WHILE_END:.*]], label %[[WHILE_BODY_PREHEADER:.*]]
; COMMON:       [[WHILE_BODY_PREHEADER]]:
; COMMON-NEXT:    br label %[[VECTOR_PH:.*]]
; COMMON:       [[VECTOR_PH]]:
; COMMON-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 15
; COMMON-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 15
; COMMON-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; COMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
; COMMON:       [[VECTOR_BODY]]:
; COMMON-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMMON-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
; COMMON-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
; COMMON-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; COMMON-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
; COMMON-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT:    [[TMP0:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
; COMMON-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP0]], ptr align 1 [[NEXT_GEP]], <16 x i1> [[ACTIVE_LANE_MASK]])
; COMMON-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 16
; COMMON-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; COMMON-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; COMMON:       [[MIDDLE_BLOCK]]:
; COMMON-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
; COMMON:       [[WHILE_END_LOOPEXIT]]:
; COMMON-NEXT:    br label %[[WHILE_END]]
; COMMON:       [[WHILE_END]]:
; COMMON-NEXT:    ret void
;
entry:
  %cmp6 = icmp eq i32 %N, 0
  br i1 %cmp6, label %while.end, label %while.body.preheader

while.body.preheader:
  br label %while.body

while.body:
  %N.addr.010 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]
  %C.addr.09 = phi ptr [ %incdec.ptr4, %while.body ], [ %C, %while.body.preheader ]
  %B.addr.08 = phi ptr [ %incdec.ptr1, %while.body ], [ %B, %while.body.preheader ]
  %A.addr.07 = phi ptr [ %incdec.ptr, %while.body ], [ %A, %while.body.preheader ]
  %incdec.ptr = getelementptr inbounds i8, ptr %A.addr.07, i32 1
  %0 = load i8, ptr %A.addr.07, align 1
  %incdec.ptr1 = getelementptr inbounds i8, ptr %B.addr.08, i32 1
  %1 = load i8, ptr %B.addr.08, align 1
  %add = add i8 %1, %0
  %incdec.ptr4 = getelementptr inbounds i8, ptr %C.addr.09, i32 1
  store i8 %add, ptr %C.addr.09, align 1
  %dec = add i32 %N.addr.010, -1
  %cmp = icmp eq i32 %dec, 0
  br i1 %cmp, label %while.end.loopexit, label %while.body

while.end.loopexit:
  br label %while.end

while.end:
  ret void
}

; This IR corresponds to this type of C-code:
;
;  void f(char *a, char *b, char * __restrict c, int N) {
;    #pragma clang loop vectorize_width(16)
;    for (int i = N; i>0; i--)
;      c[i] = a[i] + b[i];
;  }
;
; TODO: if tail-predication is requested, tail-folding isn't triggered because
; the profitability check returns "Different strides found, can't tail-predicate",
; investigate this.
define void @sgt_for_loop(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
; DEFAULT-LABEL: define void @sgt_for_loop(
; DEFAULT-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; DEFAULT-NEXT:  [[ENTRY:.*:]]
; DEFAULT-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
; DEFAULT-NEXT:    br i1 [[CMP5]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_END:label %.*]]
; DEFAULT:       [[FOR_BODY_PREHEADER]]:
; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 16
; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT:       [[VECTOR_PH]]:
; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N]], 15
; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; DEFAULT-NEXT:    [[TMP0:%.*]] = sub i32 [[N]], [[N_VEC]]
; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
; DEFAULT:       [[VECTOR_BODY]]:
; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; DEFAULT-NEXT:    [[TMP1:%.*]] = sub i32 [[N]], [[INDEX]]
; DEFAULT-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP1]]
; DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 -15
; DEFAULT-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
; DEFAULT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP1]]
; DEFAULT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 -15
; DEFAULT-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
; DEFAULT-NEXT:    [[REVERSE3:%.*]] = add <16 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD]]
; DEFAULT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP1]]
; DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 -15
; DEFAULT-NEXT:    store <16 x i8> [[REVERSE3]], ptr [[TMP8]], align 1
; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; DEFAULT-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; DEFAULT-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; DEFAULT:       [[MIDDLE_BLOCK]]:
; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
; DEFAULT-NEXT:    br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
; DEFAULT:       [[SCALAR_PH]]:
;
; CHECK-PREFER-LABEL: define void @sgt_for_loop(
; CHECK-PREFER-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; CHECK-PREFER-NEXT:  [[ENTRY:.*:]]
; CHECK-PREFER-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
; CHECK-PREFER-NEXT:    br i1 [[CMP5]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
; CHECK-PREFER:       [[FOR_BODY_PREHEADER]]:
; CHECK-PREFER-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-PREFER:       [[VECTOR_PH]]:
; CHECK-PREFER-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 15
; CHECK-PREFER-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 15
; CHECK-PREFER-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-PREFER-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-PREFER:       [[VECTOR_BODY]]:
; CHECK-PREFER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-PREFER-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
; CHECK-PREFER-NEXT:    [[TMP0:%.*]] = sub i32 [[N]], [[INDEX]]
; CHECK-PREFER-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP0]]
; CHECK-PREFER-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 -15
; CHECK-PREFER-NEXT:    [[REVERSE:%.*]] = shufflevector <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i1> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
; CHECK-PREFER-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[REVERSE]], <16 x i8> poison)
; CHECK-PREFER-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP0]]
; CHECK-PREFER-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP3]], i32 -15
; CHECK-PREFER-NEXT:    [[WIDE_MASKED_LOAD2:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP4]], <16 x i1> [[REVERSE]], <16 x i8> poison)
; CHECK-PREFER-NEXT:    [[REVERSE4:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD2]], [[WIDE_MASKED_LOAD]]
; CHECK-PREFER-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP0]]
; CHECK-PREFER-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 -15
; CHECK-PREFER-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[REVERSE4]], ptr align 1 [[TMP7]], <16 x i1> [[REVERSE]])
; CHECK-PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-PREFER-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-PREFER-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK-PREFER:       [[MIDDLE_BLOCK]]:
; CHECK-PREFER-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
; CHECK-PREFER:       [[FOR_END_LOOPEXIT]]:
; CHECK-PREFER-NEXT:    br label %[[FOR_END]]
; CHECK-PREFER:       [[FOR_END]]:
; CHECK-PREFER-NEXT:    ret void
;
; CHECK-ENABLE-TP-LABEL: define void @sgt_for_loop(
; CHECK-ENABLE-TP-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; CHECK-ENABLE-TP-NEXT:  [[ENTRY:.*:]]
; CHECK-ENABLE-TP-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
; CHECK-ENABLE-TP-NEXT:    br i1 [[CMP5]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_END:label %.*]]
; CHECK-ENABLE-TP:       [[FOR_BODY_PREHEADER]]:
; CHECK-ENABLE-TP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 16
; CHECK-ENABLE-TP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-ENABLE-TP:       [[VECTOR_PH]]:
; CHECK-ENABLE-TP-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N]], 15
; CHECK-ENABLE-TP-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
; CHECK-ENABLE-TP-NEXT:    [[TMP0:%.*]] = sub i32 [[N]], [[N_VEC]]
; CHECK-ENABLE-TP-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-ENABLE-TP:       [[VECTOR_BODY]]:
; CHECK-ENABLE-TP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ENABLE-TP-NEXT:    [[TMP1:%.*]] = sub i32 [[N]], [[INDEX]]
; CHECK-ENABLE-TP-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP1]]
; CHECK-ENABLE-TP-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 -15
; CHECK-ENABLE-TP-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
; CHECK-ENABLE-TP-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP1]]
; CHECK-ENABLE-TP-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 -15
; CHECK-ENABLE-TP-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
; CHECK-ENABLE-TP-NEXT:    [[REVERSE3:%.*]] = add <16 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD]]
; CHECK-ENABLE-TP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP1]]
; CHECK-ENABLE-TP-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 -15
; CHECK-ENABLE-TP-NEXT:    store <16 x i8> [[REVERSE3]], ptr [[TMP8]], align 1
; CHECK-ENABLE-TP-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-ENABLE-TP-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-ENABLE-TP-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK-ENABLE-TP:       [[MIDDLE_BLOCK]]:
; CHECK-ENABLE-TP-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
; CHECK-ENABLE-TP-NEXT:    br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-ENABLE-TP:       [[SCALAR_PH]]:
;
entry:
  %cmp5 = icmp sgt i32 %N, 0
  br i1 %cmp5, label %for.body.preheader, label %for.end

for.body.preheader:
  br label %for.body

for.body:
  %i.011 = phi i32 [ %dec, %for.body ], [ %N, %for.body.preheader ]
  %arrayidx = getelementptr inbounds i8, ptr %a, i32 %i.011
  %0 = load i8, ptr %arrayidx, align 1
  %arrayidx1 = getelementptr inbounds i8, ptr %b, i32 %i.011
  %1 = load i8, ptr %arrayidx1, align 1
  %add = add i8 %1, %0
  %arrayidx4 = getelementptr inbounds i8, ptr %c, i32 %i.011
  store i8 %add, ptr %arrayidx4, align 1
  %dec = add nsw i32 %i.011, -1
  %cmp = icmp sgt i32 %i.011, 1
  br i1 %cmp, label %for.body, label %for.end, !llvm.loop !1

for.end:
  ret void
}

define void @sgt_for_loop_i64(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
; DEFAULT-LABEL: define void @sgt_for_loop_i64(
; DEFAULT-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; DEFAULT-NEXT:  [[ENTRY:.*:]]
; DEFAULT-NEXT:    [[CMP14:%.*]] = icmp sgt i32 [[N]], 0
; DEFAULT-NEXT:    br i1 [[CMP14]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_COND_CLEANUP:label %.*]]
; DEFAULT:       [[FOR_BODY_PREHEADER]]:
; DEFAULT-NEXT:    [[CONV16:%.*]] = zext i32 [[N]] to i64
; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[CONV16]], 16
; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT:       [[VECTOR_PH]]:
; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[CONV16]], 15
; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 [[CONV16]], [[N_MOD_VF]]
; DEFAULT-NEXT:    [[TMP0:%.*]] = sub i64 [[CONV16]], [[N_VEC]]
; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
; DEFAULT:       [[VECTOR_BODY]]:
; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; DEFAULT-NEXT:    [[TMP1:%.*]] = sub i64 [[CONV16]], [[INDEX]]
; DEFAULT-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
; DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP2]]
; DEFAULT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 -15
; DEFAULT-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
; DEFAULT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP2]]
; DEFAULT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 -15
; DEFAULT-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
; DEFAULT-NEXT:    [[REVERSE3:%.*]] = add <16 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD]]
; DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP2]]
; DEFAULT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 -15
; DEFAULT-NEXT:    store <16 x i8> [[REVERSE3]], ptr [[TMP9]], align 1
; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; DEFAULT-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; DEFAULT-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; DEFAULT:       [[MIDDLE_BLOCK]]:
; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[CONV16]], [[N_VEC]]
; DEFAULT-NEXT:    br i1 [[CMP_N]], [[FOR_COND_CLEANUP_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
; DEFAULT:       [[SCALAR_PH]]:
;
; CHECK-PREFER-LABEL: define void @sgt_for_loop_i64(
; CHECK-PREFER-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; CHECK-PREFER-NEXT:  [[ENTRY:.*:]]
; CHECK-PREFER-NEXT:    [[CMP14:%.*]] = icmp sgt i32 [[N]], 0
; CHECK-PREFER-NEXT:    br i1 [[CMP14]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK-PREFER:       [[FOR_BODY_PREHEADER]]:
; CHECK-PREFER-NEXT:    [[CONV16:%.*]] = zext i32 [[N]] to i64
; CHECK-PREFER-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-PREFER:       [[VECTOR_PH]]:
; CHECK-PREFER-NEXT:    [[N_RND_UP:%.*]] = add i64 [[CONV16]], 15
; CHECK-PREFER-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 15
; CHECK-PREFER-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-PREFER-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-PREFER:       [[VECTOR_BODY]]:
; CHECK-PREFER-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-PREFER-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[INDEX]], i64 [[CONV16]])
; CHECK-PREFER-NEXT:    [[TMP0:%.*]] = sub i64 [[CONV16]], [[INDEX]]
; CHECK-PREFER-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; CHECK-PREFER-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP1]]
; CHECK-PREFER-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i32 -15
; CHECK-PREFER-NEXT:    [[REVERSE:%.*]] = shufflevector <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i1> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
; CHECK-PREFER-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP3]], <16 x i1> [[REVERSE]], <16 x i8> poison)
; CHECK-PREFER-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP1]]
; CHECK-PREFER-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[TMP4]], i32 -15
; CHECK-PREFER-NEXT:    [[WIDE_MASKED_LOAD2:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP5]], <16 x i1> [[REVERSE]], <16 x i8> poison)
; CHECK-PREFER-NEXT:    [[REVERSE4:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD2]], [[WIDE_MASKED_LOAD]]
; CHECK-PREFER-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP1]]
; CHECK-PREFER-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 -15
; CHECK-PREFER-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[REVERSE4]], ptr align 1 [[TMP8]], <16 x i1> [[REVERSE]])
; CHECK-PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-PREFER-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-PREFER-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK-PREFER:       [[MIDDLE_BLOCK]]:
; CHECK-PREFER-NEXT:    br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
; CHECK-PREFER:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
; CHECK-PREFER-NEXT:    br label %[[FOR_COND_CLEANUP]]
; CHECK-PREFER:       [[FOR_COND_CLEANUP]]:
; CHECK-PREFER-NEXT:    ret void
;
; CHECK-ENABLE-TP-LABEL: define void @sgt_for_loop_i64(
; CHECK-ENABLE-TP-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; CHECK-ENABLE-TP-NEXT:  [[ENTRY:.*:]]
; CHECK-ENABLE-TP-NEXT:    [[CMP14:%.*]] = icmp sgt i32 [[N]], 0
; CHECK-ENABLE-TP-NEXT:    br i1 [[CMP14]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_COND_CLEANUP:label %.*]]
; CHECK-ENABLE-TP:       [[FOR_BODY_PREHEADER]]:
; CHECK-ENABLE-TP-NEXT:    [[CONV16:%.*]] = zext i32 [[N]] to i64
; CHECK-ENABLE-TP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[CONV16]], 16
; CHECK-ENABLE-TP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-ENABLE-TP:       [[VECTOR_PH]]:
; CHECK-ENABLE-TP-NEXT:    [[N_MOD_VF:%.*]] = and i64 [[CONV16]], 15
; CHECK-ENABLE-TP-NEXT:    [[N_VEC:%.*]] = sub i64 [[CONV16]], [[N_MOD_VF]]
; CHECK-ENABLE-TP-NEXT:    [[TMP0:%.*]] = sub i64 [[CONV16]], [[N_VEC]]
; CHECK-ENABLE-TP-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-ENABLE-TP:       [[VECTOR_BODY]]:
; CHECK-ENABLE-TP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ENABLE-TP-NEXT:    [[TMP1:%.*]] = sub i64 [[CONV16]], [[INDEX]]
; CHECK-ENABLE-TP-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-ENABLE-TP-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP2]]
; CHECK-ENABLE-TP-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 -15
; CHECK-ENABLE-TP-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
; CHECK-ENABLE-TP-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP2]]
; CHECK-ENABLE-TP-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 -15
; CHECK-ENABLE-TP-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
; CHECK-ENABLE-TP-NEXT:    [[REVERSE3:%.*]] = add <16 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD]]
; CHECK-ENABLE-TP-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP2]]
; CHECK-ENABLE-TP-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 -15
; CHECK-ENABLE-TP-NEXT:    store <16 x i8> [[REVERSE3]], ptr [[TMP9]], align 1
; CHECK-ENABLE-TP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-ENABLE-TP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-ENABLE-TP-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-ENABLE-TP:       [[MIDDLE_BLOCK]]:
; CHECK-ENABLE-TP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[CONV16]], [[N_VEC]]
; CHECK-ENABLE-TP-NEXT:    br i1 [[CMP_N]], [[FOR_COND_CLEANUP_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK-ENABLE-TP:       [[SCALAR_PH]]:
;
; With -disable-mve-tail-predication=false, the target hook returns
; "preferPredicateOverEpilogue: hardware-loop is not profitable."
; so here we don't expect the tail-folding. TODO: look into this.
entry:
  %cmp14 = icmp sgt i32 %N, 0
  br i1 %cmp14, label %for.body.preheader, label %for.cond.cleanup

for.body.preheader:
  %conv16 = zext i32 %N to i64
  br label %for.body

for.cond.cleanup.loopexit:
  br label %for.cond.cleanup

for.cond.cleanup:
  ret void

for.body:
  %i.015 = phi i64 [ %dec, %for.body ], [ %conv16, %for.body.preheader ]
  %idxprom = trunc i64 %i.015 to i32
  %arrayidx = getelementptr inbounds i8, ptr %a, i32 %idxprom
  %0 = load i8, ptr %arrayidx, align 1
  %arrayidx4 = getelementptr inbounds i8, ptr %b, i32 %idxprom
  %1 = load i8, ptr %arrayidx4, align 1
  %add = add i8 %1, %0
  %arrayidx8 = getelementptr inbounds i8, ptr %c, i32 %idxprom
  store i8 %add, ptr %arrayidx8, align 1
  %dec = add nsw i64 %i.015, -1
  %cmp = icmp sgt i64 %i.015, 1
  br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !1
}

; This IR corresponds to this nested-loop:
;
;   for (int i = 0; i<N; i++)
;     for (int j = i+1; j>0; j--)
;       c[j] = a[j] + b[j];
;
; while the inner-loop looks similar to previous examples, we can't
; transform this because the inner loop because isGuarded returns
; false for the inner-loop.
;
define void @sgt_nested_loop(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
; DEFAULT-LABEL: define void @sgt_nested_loop(
; DEFAULT-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; DEFAULT-NEXT:  [[ENTRY:.*:]]
; DEFAULT-NEXT:    [[CMP21:%.*]] = icmp sgt i32 [[N]], 0
; DEFAULT-NEXT:    br i1 [[CMP21]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
; DEFAULT:       [[FOR_BODY_PREHEADER]]:
; DEFAULT-NEXT:    br label %[[FOR_BODY:.*]]
; DEFAULT:       [[FOR_COND_LOOPEXIT:.*]]:
; DEFAULT-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[ADD:%.*]], [[N]]
; DEFAULT-NEXT:    br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[FOR_BODY]]
; DEFAULT:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
; DEFAULT-NEXT:    br label %[[FOR_COND_CLEANUP]]
; DEFAULT:       [[FOR_COND_CLEANUP]]:
; DEFAULT-NEXT:    ret void
; DEFAULT:       [[FOR_BODY]]:
; DEFAULT-NEXT:    [[I_022:%.*]] = phi i32 [ [[ADD]], %[[FOR_COND_LOOPEXIT]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; DEFAULT-NEXT:    [[ADD]] = add nuw nsw i32 [[I_022]], 1
; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[ADD]], 16
; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; DEFAULT:       [[VECTOR_PH]]:
; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[ADD]], 15
; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i32 [[ADD]], [[N_MOD_VF]]
; DEFAULT-NEXT:    [[TMP0:%.*]] = sub i32 [[ADD]], [[N_VEC]]
; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
; DEFAULT:       [[VECTOR_BODY]]:
; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; DEFAULT-NEXT:    [[TMP1:%.*]] = sub i32 [[ADD]], [[INDEX]]
; DEFAULT-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP1]]
; DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 -15
; DEFAULT-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
; DEFAULT-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP1]]
; DEFAULT-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 -15
; DEFAULT-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
; DEFAULT-NEXT:    [[TMP6:%.*]] = add <16 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD]]
; DEFAULT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP1]]
; DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 -15
; DEFAULT-NEXT:    store <16 x i8> [[TMP6]], ptr [[TMP8]], align 1
; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; DEFAULT-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; DEFAULT-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; DEFAULT:       [[MIDDLE_BLOCK]]:
; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[ADD]], [[N_VEC]]
; DEFAULT-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND_LOOPEXIT]], label %[[SCALAR_PH]]
; DEFAULT:       [[SCALAR_PH]]:
;
; CHECK-PREFER-LABEL: define void @sgt_nested_loop(
; CHECK-PREFER-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; CHECK-PREFER-NEXT:  [[ENTRY:.*:]]
; CHECK-PREFER-NEXT:    [[CMP21:%.*]] = icmp sgt i32 [[N]], 0
; CHECK-PREFER-NEXT:    br i1 [[CMP21]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK-PREFER:       [[FOR_BODY_PREHEADER]]:
; CHECK-PREFER-NEXT:    br label %[[FOR_BODY:.*]]
; CHECK-PREFER:       [[FOR_COND_LOOPEXIT:.*]]:
; CHECK-PREFER-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[ADD:%.*]], [[N]]
; CHECK-PREFER-NEXT:    br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[FOR_BODY]]
; CHECK-PREFER:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
; CHECK-PREFER-NEXT:    br label %[[FOR_COND_CLEANUP]]
; CHECK-PREFER:       [[FOR_COND_CLEANUP]]:
; CHECK-PREFER-NEXT:    ret void
; CHECK-PREFER:       [[FOR_BODY]]:
; CHECK-PREFER-NEXT:    [[I_022:%.*]] = phi i32 [ [[ADD]], %[[FOR_COND_LOOPEXIT]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-PREFER-NEXT:    [[ADD]] = add nuw nsw i32 [[I_022]], 1
; CHECK-PREFER-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK-PREFER:       [[VECTOR_PH]]:
; CHECK-PREFER-NEXT:    [[N_RND_UP:%.*]] = add i32 [[ADD]], 15
; CHECK-PREFER-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[N_RND_UP]], 15
; CHECK-PREFER-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-PREFER-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-PREFER:       [[VECTOR_BODY]]:
; CHECK-PREFER-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-PREFER-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[ADD]])
; CHECK-PREFER-NEXT:    [[TMP0:%.*]] = sub i32 [[ADD]], [[INDEX]]
; CHECK-PREFER-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP0]]
; CHECK-PREFER-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 -15
; CHECK-PREFER-NEXT:    [[REVERSE:%.*]] = shufflevector <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i1> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
; CHECK-PREFER-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[REVERSE]], <16 x i8> poison)
; CHECK-PREFER-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP0]]
; CHECK-PREFER-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP3]], i32 -15
; CHECK-PREFER-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP4]], <16 x i1> [[REVERSE]], <16 x i8> poison)
; CHECK-PREFER-NEXT:    [[TMP5:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]
; CHECK-PREFER-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP0]]
; CHECK-PREFER-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 -15
; CHECK-PREFER-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP5]], ptr align 1 [[TMP7]], <16 x i1> [[REVERSE]])
; CHECK-PREFER-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-PREFER-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-PREFER-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-PREFER:       [[MIDDLE_BLOCK]]:
; CHECK-PREFER-NEXT:    br label %[[FOR_COND_LOOPEXIT]]
;
; CHECK-ENABLE-TP-LABEL: define void @sgt_nested_loop(
; CHECK-ENABLE-TP-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; CHECK-ENABLE-TP-NEXT:  [[ENTRY:.*:]]
; CHECK-ENABLE-TP-NEXT:    [[CMP21:%.*]] = icmp sgt i32 [[N]], 0
; CHECK-ENABLE-TP-NEXT:    br i1 [[CMP21]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
; CHECK-ENABLE-TP:       [[FOR_BODY_PREHEADER]]:
; CHECK-ENABLE-TP-NEXT:    br label %[[FOR_BODY:.*]]
; CHECK-ENABLE-TP:       [[FOR_COND_LOOPEXIT:.*]]:
; CHECK-ENABLE-TP-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[ADD:%.*]], [[N]]
; CHECK-ENABLE-TP-NEXT:    br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[FOR_BODY]]
; CHECK-ENABLE-TP:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
; CHECK-ENABLE-TP-NEXT:    br label %[[FOR_COND_CLEANUP]]
; CHECK-ENABLE-TP:       [[FOR_COND_CLEANUP]]:
; CHECK-ENABLE-TP-NEXT:    ret void
; CHECK-ENABLE-TP:       [[FOR_BODY]]:
; CHECK-ENABLE-TP-NEXT:    [[I_022:%.*]] = phi i32 [ [[ADD]], %[[FOR_COND_LOOPEXIT]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
; CHECK-ENABLE-TP-NEXT:    [[ADD]] = add nuw nsw i32 [[I_022]], 1
; CHECK-ENABLE-TP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[ADD]], 16
; CHECK-ENABLE-TP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-ENABLE-TP:       [[VECTOR_PH]]:
; CHECK-ENABLE-TP-NEXT:    [[N_MOD_VF:%.*]] = and i32 [[ADD]], 15
; CHECK-ENABLE-TP-NEXT:    [[N_VEC:%.*]] = sub i32 [[ADD]], [[N_MOD_VF]]
; CHECK-ENABLE-TP-NEXT:    [[TMP0:%.*]] = sub i32 [[ADD]], [[N_VEC]]
; CHECK-ENABLE-TP-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK-ENABLE-TP:       [[VECTOR_BODY]]:
; CHECK-ENABLE-TP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-ENABLE-TP-NEXT:    [[TMP1:%.*]] = sub i32 [[ADD]], [[INDEX]]
; CHECK-ENABLE-TP-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP1]]
; CHECK-ENABLE-TP-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 -15
; CHECK-ENABLE-TP-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
; CHECK-ENABLE-TP-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP1]]
; CHECK-ENABLE-TP-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 -15
; CHECK-ENABLE-TP-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
; CHECK-ENABLE-TP-NEXT:    [[TMP6:%.*]] = add <16 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD]]
; CHECK-ENABLE-TP-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP1]]
; CHECK-ENABLE-TP-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 -15
; CHECK-ENABLE-TP-NEXT:    store <16 x i8> [[TMP6]], ptr [[TMP8]], align 1
; CHECK-ENABLE-TP-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
; CHECK-ENABLE-TP-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-ENABLE-TP-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK-ENABLE-TP:       [[MIDDLE_BLOCK]]:
; CHECK-ENABLE-TP-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[ADD]], [[N_VEC]]
; CHECK-ENABLE-TP-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND_LOOPEXIT]], label %[[SCALAR_PH]]
; CHECK-ENABLE-TP:       [[SCALAR_PH]]:
;
entry:
  %cmp21 = icmp sgt i32 %N, 0
  br i1 %cmp21, label %for.body.preheader, label %for.cond.cleanup

for.body.preheader:
  br label %for.body

for.cond.loopexit:
  %exitcond = icmp eq i32 %add, %N
  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body

for.cond.cleanup.loopexit:
  br label %for.cond.cleanup

for.cond.cleanup:
  ret void

for.body:
  %i.022 = phi i32 [ %add, %for.cond.loopexit ], [ 0, %for.body.preheader ]
  %add = add nuw nsw i32 %i.022, 1
  br label %for.body4

for.body4:
  %j.020 = phi i32 [ %add, %for.body ], [ %dec, %for.body4 ]
  %arrayidx = getelementptr inbounds i8, ptr %a, i32 %j.020
  %0 = load i8, ptr %arrayidx, align 1
  %arrayidx5 = getelementptr inbounds i8, ptr %b, i32 %j.020
  %1 = load i8, ptr %arrayidx5, align 1
  %add7 = add i8 %1, %0
  %arrayidx9 = getelementptr inbounds i8, ptr %c, i32 %j.020
  store i8 %add7, ptr %arrayidx9, align 1
  %dec = add nsw i32 %j.020, -1
  %cmp2 = icmp sgt i32 %j.020, 1
  br i1 %cmp2, label %for.body4, label %for.cond.loopexit
}

attributes #0 = { nofree norecurse nounwind "target-features"="+armv8.1-m.main,+mve.fp" }

!1 = distinct !{!1, !2}
!2 = !{!"llvm.loop.vectorize.width", i32 16}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK-TF: {{.*}}