llvm/test/Transforms/LoopVectorize/RISCV/transform-narrow-interleave-to-widen-memory.ll - llvm-project - Git at Google

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
 ; RUN: opt -p loop-vectorize -mtriple riscv64 -mattr=+v -S %s | FileCheck -check-prefix=CHECK %s
 ; RUN: opt -p loop-vectorize -mtriple riscv64 -mattr=+v -S %s -tail-folding-policy=dont-fold-tail | FileCheck -check-prefix=EPILOGUE %s

 define void @load_store_interleave_group(ptr noalias %data) vscale_range(2, 1024) {
 ; CHECK-LABEL: define void @load_store_interleave_group(
 ; CHECK-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nsw i64 [[EVL_BASED_IV]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP0]], 2
 ; CHECK-NEXT:    [[WIDE_VP_LOAD:%.*]] = call <vscale x 4 x i64> @llvm.vp.load.nxv4i64.p0(ptr align 8 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VP_LOAD]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
 ; CHECK-NEXT:    [[INTERLEAVE_EVL1:%.*]] = mul nuw nsw i32 [[TMP0]], 2
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]])
 ; CHECK-NEXT:    call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr align 8 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL1]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP0]] to i64
 ; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]
 ; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 ; EPILOGUE-LABEL: define void @load_store_interleave_group(
 ; EPILOGUE-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0:[0-9]+]] {
 ; EPILOGUE-NEXT:  [[ENTRY:.*]]:
 ; EPILOGUE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; EPILOGUE-NEXT:    [[UMAX:%.*]] = shl nuw nsw i64 [[TMP0]], 1
 ; EPILOGUE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[UMAX]]
 ; EPILOGUE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; EPILOGUE:       [[VECTOR_PH]]:
 ; EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP0]]
 ; EPILOGUE-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
 ; EPILOGUE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; EPILOGUE:       [[VECTOR_BODY]]:
 ; EPILOGUE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; EPILOGUE-NEXT:    [[TMP4:%.*]] = shl nsw i64 [[INDEX]], 1
 ; EPILOGUE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP4]]
 ; EPILOGUE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP5]], align 8
 ; EPILOGUE-NEXT:    store <vscale x 2 x i64> [[WIDE_LOAD]], ptr [[TMP5]], align 8
 ; EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
 ; EPILOGUE-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; EPILOGUE-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; EPILOGUE:       [[MIDDLE_BLOCK]]:
 ; EPILOGUE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 100, [[N_VEC]]
 ; EPILOGUE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; EPILOGUE:       [[SCALAR_PH]]:
 ; EPILOGUE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; EPILOGUE-NEXT:    br label %[[LOOP:.*]]
 ; EPILOGUE:       [[LOOP]]:
 ; EPILOGUE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; EPILOGUE-NEXT:    [[MUL_2:%.*]] = shl nsw i64 [[IV]], 1
 ; EPILOGUE-NEXT:    [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[MUL_2]]
 ; EPILOGUE-NEXT:    [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8
 ; EPILOGUE-NEXT:    store i64 [[L_0]], ptr [[DATA_0]], align 8
 ; EPILOGUE-NEXT:    [[ADD_1:%.*]] = or disjoint i64 [[MUL_2]], 1
 ; EPILOGUE-NEXT:    [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[ADD_1]]
 ; EPILOGUE-NEXT:    [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 8
 ; EPILOGUE-NEXT:    store i64 [[L_1]], ptr [[DATA_1]], align 8
 ; EPILOGUE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; EPILOGUE-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
 ; EPILOGUE-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; EPILOGUE:       [[EXIT]]:
 ; EPILOGUE-NEXT:    ret void
 ;
 entry:
   br label %loop

 loop:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
   %mul.2 = shl nsw i64 %iv, 1
   %data.0 = getelementptr inbounds i64, ptr %data, i64 %mul.2
   %l.0 = load i64, ptr %data.0, align 8
   store i64 %l.0, ptr %data.0, align 8
   %add.1 = or disjoint i64 %mul.2, 1
   %data.1 = getelementptr inbounds i64, ptr %data, i64 %add.1
   %l.1 = load i64, ptr %data.1, align 8
   store i64 %l.1, ptr %data.1, align 8
   %iv.next = add nuw nsw i64 %iv, 1
   %ec = icmp eq i64 %iv.next, 100
   br i1 %ec, label %exit, label %loop

 exit:
   ret void
 }

 define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) vscale_range(2, 1024) {
 ; CHECK-LABEL: define void @interleave_group_with_countable_early_exit(
 ; CHECK-SAME: i64 [[N:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 26)
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], [[UMAX]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
 ; CHECK:       [[VECTOR_SCEVCHECK]]:
 ; CHECK-NEXT:    [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
 ; CHECK-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
 ; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[DST]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
 ; CHECK-NEXT:    br i1 [[TMP9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP13:%.*]] = select i1 [[TMP12]], i64 [[TMP1]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP13]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr { i64, i64 }, ptr [[DST]], i64 [[INDEX]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[TMP14]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[COND:%.*]] = icmp ugt i64 [[IV]], [[N]]
 ; CHECK-NEXT:    br i1 [[COND]], label %[[EXIT1:.*]], label %[[LOOP_LATCH]]
 ; CHECK:       [[EXIT1]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[LOOP_LATCH]]:
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr { i64, i64 }, ptr [[DST]], i64 [[IV]]
 ; CHECK-NEXT:    store i64 0, ptr [[GEP1]], align 8
 ; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr [[GEP1]], i64 8
 ; CHECK-NEXT:    store i64 0, ptr [[GEP2]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[N]], [[IV]]
 ; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT2:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[EXIT2]]:
 ; CHECK-NEXT:    ret void
 ;
 ; EPILOGUE-LABEL: define void @interleave_group_with_countable_early_exit(
 ; EPILOGUE-SAME: i64 [[N:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {
 ; EPILOGUE-NEXT:  [[ENTRY:.*]]:
 ; EPILOGUE-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; EPILOGUE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; EPILOGUE-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 1
 ; EPILOGUE-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 28)
 ; EPILOGUE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], [[UMAX]]
 ; EPILOGUE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
 ; EPILOGUE:       [[VECTOR_SCEVCHECK]]:
 ; EPILOGUE-NEXT:    [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
 ; EPILOGUE-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
 ; EPILOGUE-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
 ; EPILOGUE-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]]
 ; EPILOGUE-NEXT:    [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[DST]]
 ; EPILOGUE-NEXT:    [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
 ; EPILOGUE-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
 ; EPILOGUE-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
 ; EPILOGUE-NEXT:    [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
 ; EPILOGUE-NEXT:    [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
 ; EPILOGUE-NEXT:    [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
 ; EPILOGUE-NEXT:    br i1 [[TMP9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; EPILOGUE:       [[VECTOR_PH]]:
 ; EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP1]]
 ; EPILOGUE-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; EPILOGUE-NEXT:    [[TMP13:%.*]] = select i1 [[TMP12]], i64 [[TMP1]], i64 [[N_MOD_VF]]
 ; EPILOGUE-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP13]]
 ; EPILOGUE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; EPILOGUE:       [[VECTOR_BODY]]:
 ; EPILOGUE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; EPILOGUE-NEXT:    [[TMP14:%.*]] = getelementptr { i64, i64 }, ptr [[DST]], i64 [[INDEX]]
 ; EPILOGUE-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[TMP14]], align 8
 ; EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
 ; EPILOGUE-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; EPILOGUE-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; EPILOGUE:       [[MIDDLE_BLOCK]]:
 ; EPILOGUE-NEXT:    br label %[[SCALAR_PH]]
 ; EPILOGUE:       [[SCALAR_PH]]:
 ; EPILOGUE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
 ; EPILOGUE-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; EPILOGUE:       [[LOOP_HEADER]]:
 ; EPILOGUE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; EPILOGUE-NEXT:    [[COND:%.*]] = icmp ugt i64 [[IV]], [[N]]
 ; EPILOGUE-NEXT:    br i1 [[COND]], label %[[EXIT1:.*]], label %[[LOOP_LATCH]]
 ; EPILOGUE:       [[EXIT1]]:
 ; EPILOGUE-NEXT:    ret void
 ; EPILOGUE:       [[LOOP_LATCH]]:
 ; EPILOGUE-NEXT:    [[GEP1:%.*]] = getelementptr { i64, i64 }, ptr [[DST]], i64 [[IV]]
 ; EPILOGUE-NEXT:    store i64 0, ptr [[GEP1]], align 8
 ; EPILOGUE-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr [[GEP1]], i64 8
 ; EPILOGUE-NEXT:    store i64 0, ptr [[GEP2]], align 8
 ; EPILOGUE-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; EPILOGUE-NEXT:    [[CMP:%.*]] = icmp eq i64 [[N]], [[IV]]
 ; EPILOGUE-NEXT:    br i1 [[CMP]], label %[[EXIT2:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
 ; EPILOGUE:       [[EXIT2]]:
 ; EPILOGUE-NEXT:    ret void
 ;
 entry:
   br label %loop.header

 loop.header:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
   %cond = icmp ugt i64 %iv, %n
   br i1 %cond, label %exit1, label %loop.latch

 exit1:
   ret void

 loop.latch:
   %gep1 = getelementptr { i64, i64 }, ptr %dst, i64 %iv
   store i64 0, ptr %gep1, align 8
   %gep2 = getelementptr i8, ptr %gep1, i64 8
   store i64 0, ptr %gep2, align 8
   %iv.next = add i64 %iv, 1
   %cmp = icmp eq i64 %n, %iv
   br i1 %cmp, label %exit2, label %loop.header

 exit2:
   ret void
 }

 define void @load_store_interleave_group_i32(ptr noalias %data) vscale_range(2, 1024) {
 ; CHECK-LABEL: define void @load_store_interleave_group_i32(
 ; CHECK-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nsw i64 [[EVL_BASED_IV]], 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP0]], 4
 ; CHECK-NEXT:    [[WIDE_VP_LOAD:%.*]] = call <vscale x 16 x i32> @llvm.vp.load.nxv16i32.p0(ptr align 8 [[TMP2]], <vscale x 16 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VP_LOAD]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 3
 ; CHECK-NEXT:    [[INTERLEAVE_EVL1:%.*]] = mul nuw nsw i32 [[TMP0]], 4
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave4.nxv16i32(<vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]])
 ; CHECK-NEXT:    call void @llvm.vp.store.nxv16i32.p0(<vscale x 16 x i32> [[INTERLEAVED_VEC]], ptr align 8 [[TMP2]], <vscale x 16 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL1]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP0]] to i64
 ; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]
 ; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 ; EPILOGUE-LABEL: define void @load_store_interleave_group_i32(
 ; EPILOGUE-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0]] {
 ; EPILOGUE-NEXT:  [[ENTRY:.*]]:
 ; EPILOGUE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; EPILOGUE-NEXT:    [[UMAX:%.*]] = shl nuw nsw i64 [[TMP0]], 2
 ; EPILOGUE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[UMAX]]
 ; EPILOGUE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; EPILOGUE:       [[VECTOR_PH]]:
 ; EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP0]]
 ; EPILOGUE-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
 ; EPILOGUE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; EPILOGUE:       [[VECTOR_BODY]]:
 ; EPILOGUE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; EPILOGUE-NEXT:    [[TMP4:%.*]] = shl nsw i64 [[INDEX]], 2
 ; EPILOGUE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[TMP4]]
 ; EPILOGUE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 8
 ; EPILOGUE-NEXT:    store <vscale x 4 x i32> [[WIDE_LOAD]], ptr [[TMP5]], align 8
 ; EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
 ; EPILOGUE-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; EPILOGUE-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; EPILOGUE:       [[MIDDLE_BLOCK]]:
 ; EPILOGUE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 100, [[N_VEC]]
 ; EPILOGUE-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; EPILOGUE:       [[SCALAR_PH]]:
 ; EPILOGUE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; EPILOGUE-NEXT:    br label %[[LOOP:.*]]
 ; EPILOGUE:       [[LOOP]]:
 ; EPILOGUE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; EPILOGUE-NEXT:    [[MUL_2:%.*]] = shl nsw i64 [[IV]], 2
 ; EPILOGUE-NEXT:    [[DATA_0:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[MUL_2]]
 ; EPILOGUE-NEXT:    [[L_0:%.*]] = load i32, ptr [[DATA_0]], align 8
 ; EPILOGUE-NEXT:    store i32 [[L_0]], ptr [[DATA_0]], align 8
 ; EPILOGUE-NEXT:    [[ADD_1:%.*]] = or disjoint i64 [[MUL_2]], 1
 ; EPILOGUE-NEXT:    [[DATA_1:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_1]]
 ; EPILOGUE-NEXT:    [[L_1:%.*]] = load i32, ptr [[DATA_1]], align 8
 ; EPILOGUE-NEXT:    store i32 [[L_1]], ptr [[DATA_1]], align 8
 ; EPILOGUE-NEXT:    [[ADD_2:%.*]] = add i64 [[MUL_2]], 2
 ; EPILOGUE-NEXT:    [[DATA_2:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_2]]
 ; EPILOGUE-NEXT:    [[L_2:%.*]] = load i32, ptr [[DATA_2]], align 8
 ; EPILOGUE-NEXT:    store i32 [[L_2]], ptr [[DATA_2]], align 8
 ; EPILOGUE-NEXT:    [[ADD_3:%.*]] = add i64 [[MUL_2]], 3
 ; EPILOGUE-NEXT:    [[DATA_3:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_3]]
 ; EPILOGUE-NEXT:    [[L_3:%.*]] = load i32, ptr [[DATA_3]], align 8
 ; EPILOGUE-NEXT:    store i32 [[L_3]], ptr [[DATA_3]], align 8
 ; EPILOGUE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; EPILOGUE-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
 ; EPILOGUE-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
 ; EPILOGUE:       [[EXIT]]:
 ; EPILOGUE-NEXT:    ret void
 ;
 entry:
   br label %loop

 loop:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
   %mul.4 = shl nsw i64 %iv, 2
   %data.0 = getelementptr inbounds i32, ptr %data, i64 %mul.4
   %l.0 = load i32, ptr %data.0, align 8
   store i32 %l.0, ptr %data.0, align 8
   %add.1 = or disjoint i64 %mul.4, 1
   %data.1 = getelementptr inbounds i32, ptr %data, i64 %add.1
   %l.1 = load i32, ptr %data.1, align 8
   store i32 %l.1, ptr %data.1, align 8
   %add.2 = add i64 %mul.4, 2
   %data.2 = getelementptr inbounds i32, ptr %data, i64 %add.2
   %l.2 = load i32, ptr %data.2, align 8
   store i32 %l.2, ptr %data.2, align 8
   %add.3 = add i64 %mul.4, 3
   %data.3 = getelementptr inbounds i32, ptr %data, i64 %add.3
   %l.3 = load i32, ptr %data.3, align 8
   store i32 %l.3, ptr %data.3, align 8
   %iv.next = add nuw nsw i64 %iv, 1
   %ec = icmp eq i64 %iv.next, 100
   br i1 %ec, label %exit, label %loop

 exit:
   ret void
 }
	; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
	; RUN: opt -p loop-vectorize -mtriple riscv64 -mattr=+v -S %s \| FileCheck -check-prefix=CHECK %s
	; RUN: opt -p loop-vectorize -mtriple riscv64 -mattr=+v -S %s -tail-folding-policy=dont-fold-tail \| FileCheck -check-prefix=EPILOGUE %s

	define void @load_store_interleave_group(ptr noalias %data) vscale_range(2, 1024) {
	; CHECK-LABEL: define void @load_store_interleave_group(
	; CHECK-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0:[0-9]+]] {
	; CHECK-NEXT: [[ENTRY:.*:]]
	; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
	; CHECK: [[VECTOR_PH]]:
	; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
	; CHECK: [[VECTOR_BODY]]:
	; CHECK-NEXT: [[EVL_BASED_IV:%.]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.]], %[[VECTOR_BODY]] ]
	; CHECK-NEXT: [[AVL:%.]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.]], %[[VECTOR_BODY]] ]
	; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
	; CHECK-NEXT: [[TMP1:%.*]] = shl nsw i64 [[EVL_BASED_IV]], 1
	; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]
	; CHECK-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP0]], 2
	; CHECK-NEXT: [[WIDE_VP_LOAD:%.*]] = call <vscale x 4 x i64> @llvm.vp.load.nxv4i64.p0(ptr align 8 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
	; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VP_LOAD]])
	; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
	; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
	; CHECK-NEXT: [[INTERLEAVE_EVL1:%.*]] = mul nuw nsw i32 [[TMP0]], 2
	; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]])
	; CHECK-NEXT: call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr align 8 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL1]])
	; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP0]] to i64
	; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]
	; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
	; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
	; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
	; CHECK: [[MIDDLE_BLOCK]]:
	; CHECK-NEXT: br label %[[EXIT:.*]]
	; CHECK: [[EXIT]]:
	; CHECK-NEXT: ret void
	;
	; EPILOGUE-LABEL: define void @load_store_interleave_group(
	; EPILOGUE-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0:[0-9]+]] {
	; EPILOGUE-NEXT: [[ENTRY:.*]]:
	; EPILOGUE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
	; EPILOGUE-NEXT: [[UMAX:%.*]] = shl nuw nsw i64 [[TMP0]], 1
	; EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[UMAX]]
	; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.]], label %[[VECTOR_PH:.]]
	; EPILOGUE: [[VECTOR_PH]]:
	; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP0]]
	; EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
	; EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
	; EPILOGUE: [[VECTOR_BODY]]:
	; EPILOGUE-NEXT: [[INDEX:%.]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.]], %[[VECTOR_BODY]] ]
	; EPILOGUE-NEXT: [[TMP4:%.*]] = shl nsw i64 [[INDEX]], 1
	; EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP4]]
	; EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP5]], align 8
	; EPILOGUE-NEXT: store <vscale x 2 x i64> [[WIDE_LOAD]], ptr [[TMP5]], align 8
	; EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
	; EPILOGUE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
	; EPILOGUE-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
	; EPILOGUE: [[MIDDLE_BLOCK]]:
	; EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 100, [[N_VEC]]
	; EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
	; EPILOGUE: [[SCALAR_PH]]:
	; EPILOGUE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
	; EPILOGUE-NEXT: br label %[[LOOP:.*]]
	; EPILOGUE: [[LOOP]]:
	; EPILOGUE-NEXT: [[IV:%.]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.]], %[[LOOP]] ]
	; EPILOGUE-NEXT: [[MUL_2:%.*]] = shl nsw i64 [[IV]], 1
	; EPILOGUE-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[MUL_2]]
	; EPILOGUE-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8
	; EPILOGUE-NEXT: store i64 [[L_0]], ptr [[DATA_0]], align 8
	; EPILOGUE-NEXT: [[ADD_1:%.*]] = or disjoint i64 [[MUL_2]], 1
	; EPILOGUE-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[ADD_1]]
	; EPILOGUE-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 8
	; EPILOGUE-NEXT: store i64 [[L_1]], ptr [[DATA_1]], align 8
	; EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
	; EPILOGUE-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
	; EPILOGUE-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
	; EPILOGUE: [[EXIT]]:
	; EPILOGUE-NEXT: ret void
	;
	entry:
	br label %loop

	loop:
	%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
	%mul.2 = shl nsw i64 %iv, 1
	%data.0 = getelementptr inbounds i64, ptr %data, i64 %mul.2
	%l.0 = load i64, ptr %data.0, align 8
	store i64 %l.0, ptr %data.0, align 8
	%add.1 = or disjoint i64 %mul.2, 1
	%data.1 = getelementptr inbounds i64, ptr %data, i64 %add.1
	%l.1 = load i64, ptr %data.1, align 8
	store i64 %l.1, ptr %data.1, align 8
	%iv.next = add nuw nsw i64 %iv, 1
	%ec = icmp eq i64 %iv.next, 100
	br i1 %ec, label %exit, label %loop

	exit:
	ret void
	}

	define void @interleave_group_with_countable_early_exit(i64 %n, ptr %dst) vscale_range(2, 1024) {
	; CHECK-LABEL: define void @interleave_group_with_countable_early_exit(
	; CHECK-SAME: i64 [[N:%.]], ptr [[DST:%.]]) #[[ATTR0]] {
	; CHECK-NEXT: [[ENTRY:.*]]:
	; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
	; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
	; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 1
	; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 26)
	; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], [[UMAX]]
	; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.]], label %[[VECTOR_SCEVCHECK:.]]
	; CHECK: [[VECTOR_SCEVCHECK]]:
	; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
	; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
	; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
	; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]]
	; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[DST]]
	; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
	; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
	; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
	; CHECK-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
	; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
	; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
	; CHECK-NEXT: br i1 [[TMP9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
	; CHECK: [[VECTOR_PH]]:
	; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP1]]
	; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
	; CHECK-NEXT: [[TMP13:%.*]] = select i1 [[TMP12]], i64 [[TMP1]], i64 [[N_MOD_VF]]
	; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP13]]
	; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
	; CHECK: [[VECTOR_BODY]]:
	; CHECK-NEXT: [[INDEX:%.]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.]], %[[VECTOR_BODY]] ]
	; CHECK-NEXT: [[TMP14:%.*]] = getelementptr { i64, i64 }, ptr [[DST]], i64 [[INDEX]]
	; CHECK-NEXT: store <vscale x 2 x i64> zeroinitializer, ptr [[TMP14]], align 8
	; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
	; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
	; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
	; CHECK: [[MIDDLE_BLOCK]]:
	; CHECK-NEXT: br label %[[SCALAR_PH]]
	; CHECK: [[SCALAR_PH]]:
	; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
	; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
	; CHECK: [[LOOP_HEADER]]:
	; CHECK-NEXT: [[IV:%.]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.]], %[[LOOP_LATCH:.*]] ]
	; CHECK-NEXT: [[COND:%.*]] = icmp ugt i64 [[IV]], [[N]]
	; CHECK-NEXT: br i1 [[COND]], label %[[EXIT1:.*]], label %[[LOOP_LATCH]]
	; CHECK: [[EXIT1]]:
	; CHECK-NEXT: ret void
	; CHECK: [[LOOP_LATCH]]:
	; CHECK-NEXT: [[GEP1:%.*]] = getelementptr { i64, i64 }, ptr [[DST]], i64 [[IV]]
	; CHECK-NEXT: store i64 0, ptr [[GEP1]], align 8
	; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr [[GEP1]], i64 8
	; CHECK-NEXT: store i64 0, ptr [[GEP2]], align 8
	; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
	; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[N]], [[IV]]
	; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT2:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]
	; CHECK: [[EXIT2]]:
	; CHECK-NEXT: ret void
	;
	; EPILOGUE-LABEL: define void @interleave_group_with_countable_early_exit(
	; EPILOGUE-SAME: i64 [[N:%.]], ptr [[DST:%.]]) #[[ATTR0]] {
	; EPILOGUE-NEXT: [[ENTRY:.*]]:
	; EPILOGUE-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
	; EPILOGUE-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
	; EPILOGUE-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 1
	; EPILOGUE-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP2]], i64 28)
	; EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], [[UMAX]]
	; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.]], label %[[VECTOR_SCEVCHECK:.]]
	; EPILOGUE: [[VECTOR_SCEVCHECK]]:
	; EPILOGUE-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
	; EPILOGUE-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
	; EPILOGUE-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
	; EPILOGUE-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]]
	; EPILOGUE-NEXT: [[TMP4:%.*]] = icmp ult ptr [[TMP3]], [[DST]]
	; EPILOGUE-NEXT: [[TMP5:%.*]] = or i1 [[TMP4]], [[MUL_OVERFLOW]]
	; EPILOGUE-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
	; EPILOGUE-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
	; EPILOGUE-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
	; EPILOGUE-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
	; EPILOGUE-NEXT: [[TMP9:%.*]] = or i1 [[TMP5]], [[TMP8]]
	; EPILOGUE-NEXT: br i1 [[TMP9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
	; EPILOGUE: [[VECTOR_PH]]:
	; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP1]]
	; EPILOGUE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
	; EPILOGUE-NEXT: [[TMP13:%.*]] = select i1 [[TMP12]], i64 [[TMP1]], i64 [[N_MOD_VF]]
	; EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP13]]
	; EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
	; EPILOGUE: [[VECTOR_BODY]]:
	; EPILOGUE-NEXT: [[INDEX:%.]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.]], %[[VECTOR_BODY]] ]
	; EPILOGUE-NEXT: [[TMP14:%.*]] = getelementptr { i64, i64 }, ptr [[DST]], i64 [[INDEX]]
	; EPILOGUE-NEXT: store <vscale x 2 x i64> zeroinitializer, ptr [[TMP14]], align 8
	; EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
	; EPILOGUE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
	; EPILOGUE-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
	; EPILOGUE: [[MIDDLE_BLOCK]]:
	; EPILOGUE-NEXT: br label %[[SCALAR_PH]]
	; EPILOGUE: [[SCALAR_PH]]:
	; EPILOGUE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
	; EPILOGUE-NEXT: br label %[[LOOP_HEADER:.*]]
	; EPILOGUE: [[LOOP_HEADER]]:
	; EPILOGUE-NEXT: [[IV:%.]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.]], %[[LOOP_LATCH:.*]] ]
	; EPILOGUE-NEXT: [[COND:%.*]] = icmp ugt i64 [[IV]], [[N]]
	; EPILOGUE-NEXT: br i1 [[COND]], label %[[EXIT1:.*]], label %[[LOOP_LATCH]]
	; EPILOGUE: [[EXIT1]]:
	; EPILOGUE-NEXT: ret void
	; EPILOGUE: [[LOOP_LATCH]]:
	; EPILOGUE-NEXT: [[GEP1:%.*]] = getelementptr { i64, i64 }, ptr [[DST]], i64 [[IV]]
	; EPILOGUE-NEXT: store i64 0, ptr [[GEP1]], align 8
	; EPILOGUE-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr [[GEP1]], i64 8
	; EPILOGUE-NEXT: store i64 0, ptr [[GEP2]], align 8
	; EPILOGUE-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
	; EPILOGUE-NEXT: [[CMP:%.*]] = icmp eq i64 [[N]], [[IV]]
	; EPILOGUE-NEXT: br i1 [[CMP]], label %[[EXIT2:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
	; EPILOGUE: [[EXIT2]]:
	; EPILOGUE-NEXT: ret void
	;
	entry:
	br label %loop.header

	loop.header:
	%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
	%cond = icmp ugt i64 %iv, %n
	br i1 %cond, label %exit1, label %loop.latch

	exit1:
	ret void

	loop.latch:
	%gep1 = getelementptr { i64, i64 }, ptr %dst, i64 %iv
	store i64 0, ptr %gep1, align 8
	%gep2 = getelementptr i8, ptr %gep1, i64 8
	store i64 0, ptr %gep2, align 8
	%iv.next = add i64 %iv, 1
	%cmp = icmp eq i64 %n, %iv
	br i1 %cmp, label %exit2, label %loop.header

	exit2:
	ret void
	}

	define void @load_store_interleave_group_i32(ptr noalias %data) vscale_range(2, 1024) {
	; CHECK-LABEL: define void @load_store_interleave_group_i32(
	; CHECK-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0]] {
	; CHECK-NEXT: [[ENTRY:.*:]]
	; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
	; CHECK: [[VECTOR_PH]]:
	; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
	; CHECK: [[VECTOR_BODY]]:
	; CHECK-NEXT: [[EVL_BASED_IV:%.]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.]], %[[VECTOR_BODY]] ]
	; CHECK-NEXT: [[AVL:%.]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.]], %[[VECTOR_BODY]] ]
	; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
	; CHECK-NEXT: [[TMP1:%.*]] = shl nsw i64 [[EVL_BASED_IV]], 2
	; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[TMP1]]
	; CHECK-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP0]], 4
	; CHECK-NEXT: [[WIDE_VP_LOAD:%.*]] = call <vscale x 16 x i32> @llvm.vp.load.nxv16i32.p0(ptr align 8 [[TMP2]], <vscale x 16 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
	; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VP_LOAD]])
	; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
	; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
	; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
	; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 3
	; CHECK-NEXT: [[INTERLEAVE_EVL1:%.*]] = mul nuw nsw i32 [[TMP0]], 4
	; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave4.nxv16i32(<vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]])
	; CHECK-NEXT: call void @llvm.vp.store.nxv16i32.p0(<vscale x 16 x i32> [[INTERLEAVED_VEC]], ptr align 8 [[TMP2]], <vscale x 16 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL1]])
	; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP0]] to i64
	; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]
	; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
	; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
	; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
	; CHECK: [[MIDDLE_BLOCK]]:
	; CHECK-NEXT: br label %[[EXIT:.*]]
	; CHECK: [[EXIT]]:
	; CHECK-NEXT: ret void
	;
	; EPILOGUE-LABEL: define void @load_store_interleave_group_i32(
	; EPILOGUE-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0]] {
	; EPILOGUE-NEXT: [[ENTRY:.*]]:
	; EPILOGUE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
	; EPILOGUE-NEXT: [[UMAX:%.*]] = shl nuw nsw i64 [[TMP0]], 2
	; EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[UMAX]]
	; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.]], label %[[VECTOR_PH:.]]
	; EPILOGUE: [[VECTOR_PH]]:
	; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP0]]
	; EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
	; EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]
	; EPILOGUE: [[VECTOR_BODY]]:
	; EPILOGUE-NEXT: [[INDEX:%.]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.]], %[[VECTOR_BODY]] ]
	; EPILOGUE-NEXT: [[TMP4:%.*]] = shl nsw i64 [[INDEX]], 2
	; EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[TMP4]]
	; EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 8
	; EPILOGUE-NEXT: store <vscale x 4 x i32> [[WIDE_LOAD]], ptr [[TMP5]], align 8
	; EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP0]]
	; EPILOGUE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
	; EPILOGUE-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
	; EPILOGUE: [[MIDDLE_BLOCK]]:
	; EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 100, [[N_VEC]]
	; EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
	; EPILOGUE: [[SCALAR_PH]]:
	; EPILOGUE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
	; EPILOGUE-NEXT: br label %[[LOOP:.*]]
	; EPILOGUE: [[LOOP]]:
	; EPILOGUE-NEXT: [[IV:%.]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.]], %[[LOOP]] ]
	; EPILOGUE-NEXT: [[MUL_2:%.*]] = shl nsw i64 [[IV]], 2
	; EPILOGUE-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[MUL_2]]
	; EPILOGUE-NEXT: [[L_0:%.*]] = load i32, ptr [[DATA_0]], align 8
	; EPILOGUE-NEXT: store i32 [[L_0]], ptr [[DATA_0]], align 8
	; EPILOGUE-NEXT: [[ADD_1:%.*]] = or disjoint i64 [[MUL_2]], 1
	; EPILOGUE-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_1]]
	; EPILOGUE-NEXT: [[L_1:%.*]] = load i32, ptr [[DATA_1]], align 8
	; EPILOGUE-NEXT: store i32 [[L_1]], ptr [[DATA_1]], align 8
	; EPILOGUE-NEXT: [[ADD_2:%.*]] = add i64 [[MUL_2]], 2
	; EPILOGUE-NEXT: [[DATA_2:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_2]]
	; EPILOGUE-NEXT: [[L_2:%.*]] = load i32, ptr [[DATA_2]], align 8
	; EPILOGUE-NEXT: store i32 [[L_2]], ptr [[DATA_2]], align 8
	; EPILOGUE-NEXT: [[ADD_3:%.*]] = add i64 [[MUL_2]], 3
	; EPILOGUE-NEXT: [[DATA_3:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_3]]
	; EPILOGUE-NEXT: [[L_3:%.*]] = load i32, ptr [[DATA_3]], align 8
	; EPILOGUE-NEXT: store i32 [[L_3]], ptr [[DATA_3]], align 8
	; EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
	; EPILOGUE-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
	; EPILOGUE-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
	; EPILOGUE: [[EXIT]]:
	; EPILOGUE-NEXT: ret void
	;
	entry:
	br label %loop

	loop:
	%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
	%mul.4 = shl nsw i64 %iv, 2
	%data.0 = getelementptr inbounds i32, ptr %data, i64 %mul.4
	%l.0 = load i32, ptr %data.0, align 8
	store i32 %l.0, ptr %data.0, align 8
	%add.1 = or disjoint i64 %mul.4, 1
	%data.1 = getelementptr inbounds i32, ptr %data, i64 %add.1
	%l.1 = load i32, ptr %data.1, align 8
	store i32 %l.1, ptr %data.1, align 8
	%add.2 = add i64 %mul.4, 2
	%data.2 = getelementptr inbounds i32, ptr %data, i64 %add.2
	%l.2 = load i32, ptr %data.2, align 8
	store i32 %l.2, ptr %data.2, align 8
	%add.3 = add i64 %mul.4, 3
	%data.3 = getelementptr inbounds i32, ptr %data, i64 %add.3
	%l.3 = load i32, ptr %data.3, align 8
	store i32 %l.3, ptr %data.3, align 8
	%iv.next = add nuw nsw i64 %iv, 1
	%ec = icmp eq i64 %iv.next, 100
	br i1 %ec, label %exit, label %loop

	exit:
	ret void
	}