| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 6 | |
| ; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s | |
| ; if (start <u len && start - 32 <u len) | |
| ; for (int32_t i = start; i < n; i++) a[i] = a[i - 32] | |
| ; len is in [0, INT_MAX). start <u len, so start >=s 0. start - 32 <u len, so | |
| ; start - 32 doesn't wrap, and start >=s 32. So i - 32 >=s 0 on every | |
| ; iteration, and the distance between a[i] and a[i - 32] is always 32. | |
| ; i - 32 is computed in i64, so the distance is a constant and no runtime check | |
| ; is needed. | |
| define void @index_i64(ptr %a, i32 %start, i32 %n, ptr %len.p) { | |
| ; CHECK-LABEL: define void @index_i64( | |
| ; CHECK-SAME: ptr [[A:%.*]], i32 [[START:%.*]], i32 [[N:%.*]], ptr [[LEN_P:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*:]] | |
| ; CHECK-NEXT: [[LEN:%.*]] = load i32, ptr [[LEN_P]], align 4, !range [[RNG0:![0-9]+]] | |
| ; CHECK-NEXT: [[START_OK:%.*]] = icmp ult i32 [[START]], [[LEN]] | |
| ; CHECK-NEXT: [[START_M32:%.*]] = add i32 [[START]], -32 | |
| ; CHECK-NEXT: [[START_M32_OK:%.*]] = icmp ult i32 [[START_M32]], [[LEN]] | |
| ; CHECK-NEXT: [[C:%.*]] = and i1 [[START_OK]], [[START_M32_OK]] | |
| ; CHECK-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], [[EXIT:label %.*]] | |
| ; CHECK: [[LOOP_PREHEADER]]: | |
| ; CHECK-NEXT: [[START_EXT:%.*]] = zext i32 [[START]] to i64 | |
| ; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[START]], 1 | |
| ; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.smax.i32(i32 [[N]], i32 [[TMP0]]) | |
| ; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], -1 | |
| ; CHECK-NEXT: [[TMP3:%.*]] = sub i32 [[TMP2]], [[START]] | |
| ; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 | |
| ; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 1 | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP5]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP6:%.*]] = and i64 [[TMP5]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP5]], [[TMP6]] | |
| ; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[START_EXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP8:%.*]] = add nuw i64 [[START_EXT]], [[INDEX]] | |
| ; CHECK-NEXT: [[TMP9:%.*]] = add nsw i64 [[TMP8]], -32 | |
| ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP9]] | |
| ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4 | |
| ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP8]] | |
| ; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP11]], align 4 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP5]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; | |
| entry: | |
| %len = load i32, ptr %len.p, !range !0 | |
| %start.ok = icmp ult i32 %start, %len | |
| %start.m32 = add i32 %start, -32 | |
| %start.m32.ok = icmp ult i32 %start.m32, %len | |
| %c = and i1 %start.ok, %start.m32.ok | |
| br i1 %c, label %loop.preheader, label %exit | |
| loop.preheader: | |
| %start.ext = zext i32 %start to i64 | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ %start.ext, %loop.preheader ], [ %iv.next, %loop ] | |
| %idx = add nsw i64 %iv, -32 | |
| %src = getelementptr i32, ptr %a, i64 %idx | |
| %v = load i32, ptr %src | |
| %dst = getelementptr i32, ptr %a, i64 %iv | |
| store i32 %v, ptr %dst | |
| %iv.next = add nuw nsw i64 %iv, 1 | |
| %iv.next.trunc = trunc nuw i64 %iv.next to i32 | |
| %ec = icmp sge i32 %iv.next.trunc, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; i - 32 is computed in i32 and zero-extended. | |
| ; TODO: no runtime checks are needed. | |
| define void @index_zext_i32(ptr %a, i32 %start, i32 %n, ptr %len.p) { | |
| ; CHECK-LABEL: define void @index_zext_i32( | |
| ; CHECK-SAME: ptr [[A:%.*]], i32 [[START:%.*]], i32 [[N:%.*]], ptr [[LEN_P:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*:]] | |
| ; CHECK-NEXT: [[LEN:%.*]] = load i32, ptr [[LEN_P]], align 4, !range [[RNG0]] | |
| ; CHECK-NEXT: [[START_OK:%.*]] = icmp ult i32 [[START]], [[LEN]] | |
| ; CHECK-NEXT: [[START_M32:%.*]] = add i32 [[START]], -32 | |
| ; CHECK-NEXT: [[START_M32_OK:%.*]] = icmp ult i32 [[START_M32]], [[LEN]] | |
| ; CHECK-NEXT: [[C:%.*]] = and i1 [[START_OK]], [[START_M32_OK]] | |
| ; CHECK-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], [[EXIT:label %.*]] | |
| ; CHECK: [[LOOP_PREHEADER]]: | |
| ; CHECK-NEXT: [[START_EXT:%.*]] = zext i32 [[START]] to i64 | |
| ; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[START]], 1 | |
| ; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.smax.i32(i32 [[N]], i32 [[TMP0]]) | |
| ; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], -1 | |
| ; CHECK-NEXT: [[TMP3:%.*]] = sub i32 [[TMP2]], [[START]] | |
| ; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP3]] to i64 | |
| ; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 1 | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP5]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]] | |
| ; CHECK: [[VECTOR_SCEVCHECK]]: | |
| ; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[START]], 1 | |
| ; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N]], i32 [[TMP6]]) | |
| ; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[SMAX]], -1 | |
| ; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[TMP7]], [[START]] | |
| ; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[START_M32]], [[TMP8]] | |
| ; CHECK-NEXT: [[TMP10:%.*]] = icmp ult i32 [[TMP9]], [[START_M32]] | |
| ; CHECK-NEXT: br i1 [[TMP10]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]] | |
| ; CHECK: [[VECTOR_MEMCHECK]]: | |
| ; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[START_EXT]], 2 | |
| ; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[START_M32]] to i64 | |
| ; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP12]], 2 | |
| ; CHECK-NEXT: [[TMP14:%.*]] = sub i64 [[TMP11]], [[TMP13]] | |
| ; CHECK-NEXT: [[TMP15:%.*]] = sub i64 [[TMP14]], 1 | |
| ; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP15]], 15 | |
| ; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP16:%.*]] = and i64 [[TMP5]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP5]], [[TMP16]] | |
| ; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[START_EXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP18:%.*]] = add nuw i64 [[START_EXT]], [[INDEX]] | |
| ; CHECK-NEXT: [[TMP19:%.*]] = trunc nuw i64 [[TMP18]] to i32 | |
| ; CHECK-NEXT: [[TMP20:%.*]] = add nsw i32 [[TMP19]], -32 | |
| ; CHECK-NEXT: [[TMP21:%.*]] = zext i32 [[TMP20]] to i64 | |
| ; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP21]] | |
| ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4 | |
| ; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP18]] | |
| ; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP23]], align 4 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP5]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; | |
| entry: | |
| %len = load i32, ptr %len.p, !range !0 | |
| %start.ok = icmp ult i32 %start, %len | |
| %start.m32 = add i32 %start, -32 | |
| %start.m32.ok = icmp ult i32 %start.m32, %len | |
| %c = and i1 %start.ok, %start.m32.ok | |
| br i1 %c, label %loop.preheader, label %exit | |
| loop.preheader: | |
| %start.ext = zext i32 %start to i64 | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ %start.ext, %loop.preheader ], [ %iv.next, %loop ] | |
| %iv.trunc = trunc nuw i64 %iv to i32 | |
| %idx.narrow = add nsw i32 %iv.trunc, -32 | |
| %idx = zext i32 %idx.narrow to i64 | |
| %src = getelementptr i32, ptr %a, i64 %idx | |
| %v = load i32, ptr %src | |
| %dst = getelementptr i32, ptr %a, i64 %iv | |
| store i32 %v, ptr %dst | |
| %iv.next = add nuw nsw i64 %iv, 1 | |
| %iv.next.trunc = trunc nuw i64 %iv.next to i32 | |
| %ec = icmp sge i32 %iv.next.trunc, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; [0, INT_MAX) | |
| !0 = !{i32 0, i32 2147483647} |