| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5 | |
| ; RUN: opt < %s -passes=loop-vectorize -mtriple=riscv64 -mattr=+v,+zvl256b -S | FileCheck %s | |
| ; Test for https://github.com/llvm/llvm-project/issues/199640 | |
| ; convertToStridedAccesses uses the canonical IV type (i32) for the stride | |
| ; constant, but the pointer stride exceeds INT32_MAX on rv64 with a narrow IV. | |
| ; TODO: Should form a strided load with SCEV predicates. | |
| define void @stride_sext_exceeds_i32_max(ptr noalias readonly %src, ptr noalias %dst, i16 %start) vscale_range(4, 1024) { | |
| ; CHECK-LABEL: define void @stride_sext_exceeds_i32_max( | |
| ; CHECK-SAME: ptr noalias readonly [[SRC:%.*]], ptr noalias [[DST:%.*]], i16 [[START:%.*]]) #[[ATTR0:[0-9]+]] { | |
| ; CHECK-NEXT: [[ENTRY:.*:]] | |
| ; CHECK-NEXT: [[TMP0:%.*]] = sub i16 -1, [[START]] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[TMP0]] to i32 | |
| ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 1 | |
| ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[DST]], i64 0 | |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer | |
| ; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i16> @llvm.stepvector.nxv8i16() | |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[START]], i64 0 | |
| ; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 8 x i16> [[BROADCAST_SPLATINSERT1]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer | |
| ; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 8 x i16> [[BROADCAST_SPLAT2]], [[TMP3]] | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 8 x i16> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP2]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true) | |
| ; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i16 | |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[TMP5]], i64 0 | |
| ; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 8 x i16> [[BROADCAST_SPLATINSERT3]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer | |
| ; CHECK-NEXT: [[TMP6:%.*]] = sext <vscale x 8 x i16> [[VEC_IND]] to <vscale x 8 x i64> | |
| ; CHECK-NEXT: [[TMP7:%.*]] = mul nsw <vscale x 8 x i64> [[TMP6]], splat (i64 3000000000) | |
| ; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr i8, ptr [[SRC]], <vscale x 8 x i64> [[TMP7]] | |
| ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 8 x i8> @llvm.vp.gather.nxv8i8.nxv8p0(<vscale x 8 x ptr> align 1 [[WIDE_GEP]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP4]]) | |
| ; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[WIDE_MASKED_GATHER]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP4]]) | |
| ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP4]] | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 8 x i16> [[VEC_IND]], [[BROADCAST_SPLAT4]] | |
| ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[AVL_NEXT]], 0 | |
| ; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: br label %[[EXIT:.*]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i16 [ %start, %entry ], [ %iv.next, %loop ] | |
| %iv.ext = sext i16 %iv to i64 | |
| %offset = mul nsw i64 %iv.ext, 3000000000 | |
| %gep = getelementptr i8, ptr %src, i64 %offset | |
| %val = load i8, ptr %gep, align 1 | |
| store i8 %val, ptr %dst, align 1 | |
| %iv.next = add i16 %iv, 1 | |
| %cmp = icmp ne i16 %iv.next, 0 | |
| br i1 %cmp, label %loop, label %exit | |
| exit: | |
| ret void | |
| } | |
| define void @stride_zext_exceeds_i32_max(ptr noalias readonly %src, ptr noalias %dst, i16 %start) vscale_range(4, 1024) { | |
| ; CHECK-LABEL: define void @stride_zext_exceeds_i32_max( | |
| ; CHECK-SAME: ptr noalias readonly [[SRC:%.*]], ptr noalias [[DST:%.*]], i16 [[START:%.*]]) #[[ATTR0]] { | |
| ; CHECK-NEXT: [[ENTRY:.*:]] | |
| ; CHECK-NEXT: [[TMP0:%.*]] = sub i16 -1, [[START]] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[TMP0]] to i32 | |
| ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 1 | |
| ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[START]] to i64 | |
| ; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw i64 [[TMP3]], 3000000000 | |
| ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP4]] | |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[DST]], i64 0 | |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP2]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 8, i1 true) | |
| ; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[INDEX]] to i64 | |
| ; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 3000000000 | |
| ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP5]], i64 [[TMP8]] | |
| ; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 8 x i8> @llvm.experimental.vp.strided.load.nxv8i8.p0.i64(ptr align 1 [[TMP9]], i64 3000000000, <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]]) | |
| ; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP10]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]]) | |
| ; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add nuw i32 [[TMP6]], [[INDEX]] | |
| ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]] | |
| ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i32 [[AVL_NEXT]], 0 | |
| ; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: br label %[[EXIT:.*]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i16 [ %start, %entry ], [ %iv.next, %loop ] | |
| %iv.ext = zext i16 %iv to i64 | |
| %offset = mul nsw i64 %iv.ext, 3000000000 | |
| %gep = getelementptr i8, ptr %src, i64 %offset | |
| %val = load i8, ptr %gep, align 1 | |
| store i8 %val, ptr %dst, align 1 | |
| %iv.next = add i16 %iv, 1 | |
| %cmp = icmp ne i16 %iv.next, 0 | |
| br i1 %cmp, label %loop, label %exit | |
| exit: | |
| ret void | |
| } |