llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -p loop-vectorize -S %s | FileCheck %s

target triple = "x86_64-pc-windows-gnu"

define i8 @low_trip_count_via_profile_info_with_live_out(ptr align 16 %start, ptr align 16 %end, ptr noalias %src) #0 {
; CHECK-LABEL: define i8 @low_trip_count_via_profile_info_with_live_out(
; CHECK-SAME: ptr align 16 [[START:%.*]], ptr align 16 [[END:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT:  [[LOOP:.*]]:
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[IV_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[PTR:%.*]] = phi ptr [ [[START]], %[[LOOP]] ], [ [[PTR_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[IV_NEXT]]
; CHECK-NEXT:    [[L:%.*]] = load i8, ptr [[GEP_SRC]], align 1
; CHECK-NEXT:    [[PTR_NEXT]] = getelementptr i8, ptr [[PTR]], i64 1
; CHECK-NEXT:    store i8 [[L]], ptr [[PTR]], align 1
; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq ptr [[PTR]], [[END]]
; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[EXIT1:.*]], label %[[VECTOR_BODY]], !prof [[PROF0:![0-9]+]]
; CHECK:       [[EXIT1]]:
; CHECK-NEXT:    [[L_LCSSA:%.*]] = phi i8 [ [[L]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    ret i8 [[L_LCSSA]]
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %ptr = phi ptr [ %start, %entry ], [ %ptr.next, %loop ]
  %iv.next = add i64 %iv, 1
  %gep.src = getelementptr i8, ptr %src, i64 %iv.next
  %l = load i8, ptr %gep.src, align 1
  %ptr.next = getelementptr i8, ptr %ptr, i64 1
  store i8 %l, ptr %ptr, align 1
  %exitcond = icmp eq ptr %ptr, %end
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret i8 %l
}

define ptr @low_trip_count_via_profile_info_with_iv_live_out(ptr align 16 %start, ptr align 16 %end, ptr noalias %src) #0 {
; CHECK-LABEL: define ptr @low_trip_count_via_profile_info_with_iv_live_out(
; CHECK-SAME: ptr align 16 [[START:%.*]], ptr align 16 [[END:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[START2:%.*]] = ptrtoint ptr [[START]] to i64
; CHECK-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END]] to i64
; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[END1]], 1
; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP1]], 63
; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 64
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP1]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <64 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <64 x i64> [[BROADCAST_SPLATINSERT]], <64 x i64> poison, <64 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[VEC_IV:%.*]] = phi <64 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31, i64 32, i64 33, i64 34, i64 35, i64 36, i64 37, i64 38, i64 39, i64 40, i64 41, i64 42, i64 43, i64 44, i64 45, i64 46, i64 47, i64 48, i64 49, i64 50, i64 51, i64 52, i64 53, i64 54, i64 55, i64 56, i64 57, i64 58, i64 59, i64 60, i64 61, i64 62, i64 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]
; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <64 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP4]]
; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[TMP5]], <64 x i1> [[TMP3]], <64 x i8> poison)
; CHECK-NEXT:    call void @llvm.masked.store.v64i8.p0(<64 x i8> [[WIDE_MASKED_LOAD]], ptr align 1 [[NEXT_GEP]], <64 x i1> [[TMP3]])
; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 64
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <64 x i64> [[VEC_IV]], splat (i64 64)
; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret ptr [[TMP2]]
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %ptr = phi ptr [ %start, %entry ], [ %ptr.next, %loop ]
  %iv.next = add i64 %iv, 1
  %gep.src = getelementptr i8, ptr %src, i64 %iv.next
  %l = load i8, ptr %gep.src, align 1
  %ptr.next = getelementptr i8, ptr %ptr, i64 1
  store i8 %l, ptr %ptr, align 1
  %exitcond = icmp eq ptr %ptr, %end
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret ptr %ptr.next
}

define void @low_trip_count_via_profile_info(ptr align 16 %start, ptr align 16 %end, ptr noalias %src) #0 {
; CHECK-LABEL: define void @low_trip_count_via_profile_info(
; CHECK-SAME: ptr align 16 [[START:%.*]], ptr align 16 [[END:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[START2:%.*]] = ptrtoint ptr [[START]] to i64
; CHECK-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END]] to i64
; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[END1]], 1
; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP1]], 63
; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 64
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <64 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <64 x i64> [[BROADCAST_SPLATINSERT]], <64 x i64> poison, <64 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[VEC_IV:%.*]] = phi <64 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31, i64 32, i64 33, i64 34, i64 35, i64 36, i64 37, i64 38, i64 39, i64 40, i64 41, i64 42, i64 43, i64 44, i64 45, i64 46, i64 47, i64 48, i64 49, i64 50, i64 51, i64 52, i64 53, i64 54, i64 55, i64 56, i64 57, i64 58, i64 59, i64 60, i64 61, i64 62, i64 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]
; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule <64 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[TMP4]], <64 x i1> [[TMP2]], <64 x i8> poison)
; CHECK-NEXT:    call void @llvm.masked.store.v64i8.p0(<64 x i8> [[WIDE_MASKED_LOAD]], ptr align 1 [[NEXT_GEP]], <64 x i1> [[TMP2]])
; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 64
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <64 x i64> [[VEC_IV]], splat (i64 64)
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %ptr = phi ptr [ %start, %entry ], [ %ptr.next, %loop ]
  %iv.next = add i64 %iv, 1
  %gep.src = getelementptr i8, ptr %src, i64 %iv.next
  %l = load i8, ptr %gep.src, align 1
  %ptr.next = getelementptr i8, ptr %ptr, i64 1
  store i8 %l, ptr %ptr, align 1
  %exitcond = icmp eq ptr %ptr, %end
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

attributes #0 = { "prefer-vector-width"="512" "target-cpu"="tigerlake" }

!0 = !{!"branch_weights", i32 169702549, i32 116357656}