; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -S -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true -enable-mem-access-versioning=true -force-target-supports-masked-memory-ops < %s | FileCheck %s --check-prefix=VF4UF1
; RUN: opt -S -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -enable-interleaved-mem-accesses=true -enable-mem-access-versioning=true -force-target-supports-masked-memory-ops < %s | FileCheck %s --check-prefix=VF1UF2
; RUN: opt -S -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=2 -enable-interleaved-mem-accesses=true -enable-mem-access-versioning=true -force-target-supports-masked-memory-ops < %s | FileCheck %s --check-prefix=VF2UF2

define void @f(ptr noalias %p, ptr noalias %q, ptr noalias %dst, i64 %stride) {
; VF4UF1-LABEL: define void @f(
; VF4UF1-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]], ptr noalias [[DST:%.*]], i64 [[STRIDE:%.*]]) {
; VF4UF1-NEXT:  [[ENTRY:.*:]]
; VF4UF1-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; VF4UF1:       [[VECTOR_SCEVCHECK]]:
; VF4UF1-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; VF4UF1-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VF4UF1:       [[VECTOR_PH]]:
; VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
; VF4UF1:       [[VECTOR_BODY]]:
; VF4UF1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4UF1-NEXT:    [[TMP0:%.*]] = shl nuw nsw i64 [[INDEX]], 3
; VF4UF1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP0]]
; VF4UF1-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
; VF4UF1-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; VF4UF1-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
; VF4UF1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [4 x i8], ptr [[Q]], i64 [[INDEX]]
; VF4UF1-NEXT:    store <4 x i32> [[WIDE_MASKED_GATHER]], ptr [[TMP2]], align 4
; VF4UF1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[INDEX]]
; VF4UF1-NEXT:    store <4 x i32> zeroinitializer, ptr [[TMP3]], align 4
; VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4UF1-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; VF4UF1-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF4UF1:       [[MIDDLE_BLOCK]]:
; VF4UF1-NEXT:    br label %[[EXIT:.*]]
; VF4UF1:       [[SCALAR_PH]]:
; VF4UF1-NEXT:    br label %[[LOOP_HEADER:.*]]
; VF4UF1:       [[LOOP_HEADER]]:
; VF4UF1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; VF4UF1-NEXT:    [[ARRAYIDX_IDX:%.*]] = shl nuw nsw i64 [[IV]], 3
; VF4UF1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[ARRAYIDX_IDX]]
; VF4UF1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; VF4UF1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX]], i64 4
; VF4UF1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
; VF4UF1-NEXT:    [[MUL3:%.*]] = mul nsw i64 [[IV]], [[STRIDE]]
; VF4UF1-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [4 x i8], ptr [[Q]], i64 [[MUL3]]
; VF4UF1-NEXT:    store i32 [[TMP5]], ptr [[ARRAYIDX4]], align 4
; VF4UF1-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[TMP5]], 0
; VF4UF1-NEXT:    br i1 [[CMP5]], label %[[IF_THEN:.*]], label %[[LOOP_LATCH]]
; VF4UF1:       [[IF_THEN]]:
; VF4UF1-NEXT:    [[REM:%.*]] = srem i64 3, [[STRIDE]]
; VF4UF1-NEXT:    [[TMP7:%.*]] = trunc nuw nsw i64 [[REM]] to i32
; VF4UF1-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP6]], [[TMP7]]
; VF4UF1-NEXT:    br label %[[LOOP_LATCH]]
; VF4UF1:       [[LOOP_LATCH]]:
; VF4UF1-NEXT:    [[T_0:%.*]] = phi i32 [ [[TMP8]], %[[IF_THEN]] ], [ 0, %[[LOOP_HEADER]] ]
; VF4UF1-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[IV]]
; VF4UF1-NEXT:    store i32 [[T_0]], ptr [[ARRAYIDX10]], align 4
; VF4UF1-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; VF4UF1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
; VF4UF1-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
; VF4UF1:       [[EXIT]]:
; VF4UF1-NEXT:    ret void
;
; VF1UF2-LABEL: define void @f(
; VF1UF2-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]], ptr noalias [[DST:%.*]], i64 [[STRIDE:%.*]]) {
; VF1UF2-NEXT:  [[ENTRY:.*:]]
; VF1UF2-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; VF1UF2:       [[VECTOR_SCEVCHECK]]:
; VF1UF2-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; VF1UF2-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VF1UF2:       [[VECTOR_PH]]:
; VF1UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
; VF1UF2:       [[VECTOR_BODY]]:
; VF1UF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF1UF2-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 1
; VF1UF2-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 3
; VF1UF2-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 3
; VF1UF2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP1]]
; VF1UF2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP2]]
; VF1UF2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4
; VF1UF2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4
; VF1UF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x i8], ptr [[Q]], i64 [[INDEX]]
; VF1UF2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [4 x i8], ptr [[Q]], i64 [[TMP0]]
; VF1UF2-NEXT:    store i32 [[TMP5]], ptr [[TMP7]], align 4
; VF1UF2-NEXT:    store i32 [[TMP6]], ptr [[TMP8]], align 4
; VF1UF2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[INDEX]]
; VF1UF2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[TMP0]]
; VF1UF2-NEXT:    store i32 0, ptr [[TMP9]], align 4
; VF1UF2-NEXT:    store i32 0, ptr [[TMP10]], align 4
; VF1UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; VF1UF2-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; VF1UF2-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF1UF2:       [[MIDDLE_BLOCK]]:
; VF1UF2-NEXT:    br label %[[EXIT:.*]]
; VF1UF2:       [[SCALAR_PH]]:
; VF1UF2-NEXT:    br label %[[LOOP_HEADER:.*]]
; VF1UF2:       [[LOOP_HEADER]]:
; VF1UF2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; VF1UF2-NEXT:    [[ARRAYIDX_IDX:%.*]] = shl nuw nsw i64 [[IV]], 3
; VF1UF2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[ARRAYIDX_IDX]]
; VF1UF2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; VF1UF2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX]], i64 4
; VF1UF2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
; VF1UF2-NEXT:    [[MUL3:%.*]] = mul nsw i64 [[IV]], [[STRIDE]]
; VF1UF2-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [4 x i8], ptr [[Q]], i64 [[MUL3]]
; VF1UF2-NEXT:    store i32 [[TMP12]], ptr [[ARRAYIDX4]], align 4
; VF1UF2-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[TMP12]], 0
; VF1UF2-NEXT:    br i1 [[CMP5]], label %[[IF_THEN:.*]], label %[[LOOP_LATCH]]
; VF1UF2:       [[IF_THEN]]:
; VF1UF2-NEXT:    [[REM:%.*]] = srem i64 3, [[STRIDE]]
; VF1UF2-NEXT:    [[TMP14:%.*]] = trunc nuw nsw i64 [[REM]] to i32
; VF1UF2-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP13]], [[TMP14]]
; VF1UF2-NEXT:    br label %[[LOOP_LATCH]]
; VF1UF2:       [[LOOP_LATCH]]:
; VF1UF2-NEXT:    [[T_0:%.*]] = phi i32 [ [[TMP15]], %[[IF_THEN]] ], [ 0, %[[LOOP_HEADER]] ]
; VF1UF2-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[IV]]
; VF1UF2-NEXT:    store i32 [[T_0]], ptr [[ARRAYIDX10]], align 4
; VF1UF2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; VF1UF2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
; VF1UF2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
; VF1UF2:       [[EXIT]]:
; VF1UF2-NEXT:    ret void
;
; VF2UF2-LABEL: define void @f(
; VF2UF2-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]], ptr noalias [[DST:%.*]], i64 [[STRIDE:%.*]]) {
; VF2UF2-NEXT:  [[ENTRY:.*:]]
; VF2UF2-NEXT:    br label %[[VECTOR_SCEVCHECK:.*]]
; VF2UF2:       [[VECTOR_SCEVCHECK]]:
; VF2UF2-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; VF2UF2-NEXT:    br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VF2UF2:       [[VECTOR_PH]]:
; VF2UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
; VF2UF2:       [[VECTOR_BODY]]:
; VF2UF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2UF2-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 2
; VF2UF2-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 3
; VF2UF2-NEXT:    [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 3
; VF2UF2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP1]]
; VF2UF2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP2]]
; VF2UF2-NEXT:    [[WIDE_VEC:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
; VF2UF2-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = shufflevector <4 x i32> [[WIDE_VEC]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
; VF2UF2-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <4 x i32> [[WIDE_VEC]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
; VF2UF2-NEXT:    [[WIDE_VEC2:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
; VF2UF2-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = shufflevector <4 x i32> [[WIDE_VEC2]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
; VF2UF2-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <4 x i32> [[WIDE_VEC2]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
; VF2UF2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[Q]], i64 [[INDEX]]
; VF2UF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 2
; VF2UF2-NEXT:    store <2 x i32> [[WIDE_MASKED_GATHER]], ptr [[TMP5]], align 4
; VF2UF2-NEXT:    store <2 x i32> [[WIDE_MASKED_GATHER3]], ptr [[TMP6]], align 4
; VF2UF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[INDEX]]
; VF2UF2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP7]], i64 2
; VF2UF2-NEXT:    store <2 x i32> zeroinitializer, ptr [[TMP7]], align 4
; VF2UF2-NEXT:    store <2 x i32> zeroinitializer, ptr [[TMP8]], align 4
; VF2UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF2UF2-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; VF2UF2-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; VF2UF2:       [[MIDDLE_BLOCK]]:
; VF2UF2-NEXT:    br label %[[EXIT:.*]]
; VF2UF2:       [[SCALAR_PH]]:
; VF2UF2-NEXT:    br label %[[LOOP_HEADER:.*]]
; VF2UF2:       [[LOOP_HEADER]]:
; VF2UF2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; VF2UF2-NEXT:    [[ARRAYIDX_IDX:%.*]] = shl nuw nsw i64 [[IV]], 3
; VF2UF2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[ARRAYIDX_IDX]]
; VF2UF2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
; VF2UF2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX]], i64 4
; VF2UF2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
; VF2UF2-NEXT:    [[MUL3:%.*]] = mul nsw i64 [[IV]], [[STRIDE]]
; VF2UF2-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [4 x i8], ptr [[Q]], i64 [[MUL3]]
; VF2UF2-NEXT:    store i32 [[TMP10]], ptr [[ARRAYIDX4]], align 4
; VF2UF2-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[TMP10]], 0
; VF2UF2-NEXT:    br i1 [[CMP5]], label %[[IF_THEN:.*]], label %[[LOOP_LATCH]]
; VF2UF2:       [[IF_THEN]]:
; VF2UF2-NEXT:    [[REM:%.*]] = srem i64 3, [[STRIDE]]
; VF2UF2-NEXT:    [[TMP12:%.*]] = trunc nuw nsw i64 [[REM]] to i32
; VF2UF2-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP11]], [[TMP12]]
; VF2UF2-NEXT:    br label %[[LOOP_LATCH]]
; VF2UF2:       [[LOOP_LATCH]]:
; VF2UF2-NEXT:    [[T_0:%.*]] = phi i32 [ [[TMP13]], %[[IF_THEN]] ], [ 0, %[[LOOP_HEADER]] ]
; VF2UF2-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[IV]]
; VF2UF2-NEXT:    store i32 [[T_0]], ptr [[ARRAYIDX10]], align 4
; VF2UF2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; VF2UF2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
; VF2UF2-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
; VF2UF2:       [[EXIT]]:
; VF2UF2-NEXT:    ret void
;
entry:
  br label %loop.header

loop.header:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %arrayidx.idx = shl nuw nsw i64 %iv, 3
  %arrayidx = getelementptr inbounds nuw i8, ptr %p, i64 %arrayidx.idx
  %0 = load i32, ptr %arrayidx, align 4
  %arrayidx2 = getelementptr inbounds nuw i8, ptr %arrayidx, i64 4
  %1 = load i32, ptr %arrayidx2, align 4
  %mul3 = mul nsw i64 %iv, %stride
  %arrayidx4 = getelementptr inbounds [4 x i8], ptr %q, i64 %mul3
  store i32 %0, ptr %arrayidx4, align 4
  %cmp5 = icmp sgt i32 %0, 0
  br i1 %cmp5, label %if.then, label %loop.latch

if.then:
  %rem = srem i64 3, %stride
  %2 = trunc nuw nsw i64 %rem to i32
  %3 = mul i32 %1, %2
  br label %loop.latch

loop.latch:
  %t.0 = phi i32 [ %3, %if.then ], [ 0, %loop.header ]
  %arrayidx10 = getelementptr inbounds nuw [4 x i8], ptr %dst, i64 %iv
  store i32 %t.0, ptr %arrayidx10, align 4
  %iv.next = add nuw nsw i64 %iv, 1
  %exitcond.not = icmp eq i64 %iv.next, 1024
  br i1 %exitcond.not, label %exit, label %loop.header

exit:
  ret void
}