blob: 711bfee760eb31cb551e8d7a1cfcc810a4c65f7c [file] [edit]
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s
; if (start <u len && start - 32 <u len)
; for (int32_t i = start; i < n; i++) a[i] = a[i - 32]
; len is in [0, INT_MAX). start <u len, so start >=s 0. start - 32 <u len, so
; start - 32 doesn't wrap, and start >=s 32. So i - 32 >=s 0 on every
; iteration, and the distance between a[i] and a[i - 32] is always 32.
; i - 32 is computed in i64, so the distance is a constant and no runtime check
; is needed.
define void @index_i64(ptr %a, i32 %start, i32 %n, ptr %len.p) {
; CHECK-LABEL: define void @index_i64(
; CHECK-SAME: ptr [[A:%.*]], i32 [[START:%.*]], i32 [[N:%.*]], ptr [[LEN_P:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[LEN:%.*]] = load i32, ptr [[LEN_P]], align 4, !range [[RNG0:![0-9]+]]
; CHECK-NEXT: [[START_OK:%.*]] = icmp ult i32 [[START]], [[LEN]]
; CHECK-NEXT: [[START_M32:%.*]] = add i32 [[START]], -32
; CHECK-NEXT: [[START_M32_OK:%.*]] = icmp ult i32 [[START_M32]], [[LEN]]
; CHECK-NEXT: [[C:%.*]] = and i1 [[START_OK]], [[START_M32_OK]]
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], [[EXIT:label %.*]]
; CHECK: [[LOOP_PREHEADER]]:
; CHECK-NEXT: [[START_EXT:%.*]] = zext i32 [[START]] to i64
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[START]], 1
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.smax.i32(i32 [[N]], i32 [[TMP0]])
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], -1
; CHECK-NEXT: [[TMP3:%.*]] = sub i32 [[TMP2]], [[START]]
; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP5]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP6:%.*]] = and i64 [[TMP5]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP5]], [[TMP6]]
; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[START_EXT]], [[N_VEC]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = add nuw i64 [[START_EXT]], [[INDEX]]
; CHECK-NEXT: [[TMP9:%.*]] = add nsw i64 [[TMP8]], -32
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP9]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP8]]
; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP11]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP5]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
;
entry:
%len = load i32, ptr %len.p, !range !0
%start.ok = icmp ult i32 %start, %len
%start.m32 = add i32 %start, -32
%start.m32.ok = icmp ult i32 %start.m32, %len
%c = and i1 %start.ok, %start.m32.ok
br i1 %c, label %loop.preheader, label %exit
loop.preheader:
%start.ext = zext i32 %start to i64
br label %loop
loop:
%iv = phi i64 [ %start.ext, %loop.preheader ], [ %iv.next, %loop ]
%idx = add nsw i64 %iv, -32
%src = getelementptr i32, ptr %a, i64 %idx
%v = load i32, ptr %src
%dst = getelementptr i32, ptr %a, i64 %iv
store i32 %v, ptr %dst
%iv.next = add nuw nsw i64 %iv, 1
%iv.next.trunc = trunc nuw i64 %iv.next to i32
%ec = icmp sge i32 %iv.next.trunc, %n
br i1 %ec, label %exit, label %loop
exit:
ret void
}
; i - 32 is computed in i32 and zero-extended.
; TODO: no runtime checks are needed.
define void @index_zext_i32(ptr %a, i32 %start, i32 %n, ptr %len.p) {
; CHECK-LABEL: define void @index_zext_i32(
; CHECK-SAME: ptr [[A:%.*]], i32 [[START:%.*]], i32 [[N:%.*]], ptr [[LEN_P:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[LEN:%.*]] = load i32, ptr [[LEN_P]], align 4, !range [[RNG0]]
; CHECK-NEXT: [[START_OK:%.*]] = icmp ult i32 [[START]], [[LEN]]
; CHECK-NEXT: [[START_M32:%.*]] = add i32 [[START]], -32
; CHECK-NEXT: [[START_M32_OK:%.*]] = icmp ult i32 [[START_M32]], [[LEN]]
; CHECK-NEXT: [[C:%.*]] = and i1 [[START_OK]], [[START_M32_OK]]
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], [[EXIT:label %.*]]
; CHECK: [[LOOP_PREHEADER]]:
; CHECK-NEXT: [[START_EXT:%.*]] = zext i32 [[START]] to i64
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[START]], 1
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.smax.i32(i32 [[N]], i32 [[TMP0]])
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], -1
; CHECK-NEXT: [[TMP3:%.*]] = sub i32 [[TMP2]], [[START]]
; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 1
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP5]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[START]], 1
; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N]], i32 [[TMP6]])
; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[SMAX]], -1
; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[TMP7]], [[START]]
; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[START_M32]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp ult i32 [[TMP9]], [[START_M32]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]]
; CHECK: [[VECTOR_MEMCHECK]]:
; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[START_EXT]], 2
; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[START_M32]] to i64
; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP12]], 2
; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[TMP11]], [[TMP13]]
; CHECK-NEXT: [[TMP15:%.*]] = sub i64 [[TMP14]], 1
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP15]], 15
; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP16:%.*]] = and i64 [[TMP5]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP5]], [[TMP16]]
; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[START_EXT]], [[N_VEC]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP18:%.*]] = add nuw i64 [[START_EXT]], [[INDEX]]
; CHECK-NEXT: [[TMP19:%.*]] = trunc nuw i64 [[TMP18]] to i32
; CHECK-NEXT: [[TMP20:%.*]] = add nsw i32 [[TMP19]], -32
; CHECK-NEXT: [[TMP21:%.*]] = zext i32 [[TMP20]] to i64
; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP21]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP18]]
; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP23]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP5]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
;
entry:
%len = load i32, ptr %len.p, !range !0
%start.ok = icmp ult i32 %start, %len
%start.m32 = add i32 %start, -32
%start.m32.ok = icmp ult i32 %start.m32, %len
%c = and i1 %start.ok, %start.m32.ok
br i1 %c, label %loop.preheader, label %exit
loop.preheader:
%start.ext = zext i32 %start to i64
br label %loop
loop:
%iv = phi i64 [ %start.ext, %loop.preheader ], [ %iv.next, %loop ]
%iv.trunc = trunc nuw i64 %iv to i32
%idx.narrow = add nsw i32 %iv.trunc, -32
%idx = zext i32 %idx.narrow to i64
%src = getelementptr i32, ptr %a, i64 %idx
%v = load i32, ptr %src
%dst = getelementptr i32, ptr %a, i64 %iv
store i32 %v, ptr %dst
%iv.next = add nuw nsw i64 %iv, 1
%iv.next.trunc = trunc nuw i64 %iv.next to i32
%ec = icmp sge i32 %iv.next.trunc, %n
br i1 %ec, label %exit, label %loop
exit:
ret void
}
; [0, INT_MAX)
!0 = !{i32 0, i32 2147483647}