llvm/test/Transforms/LoopVectorize/outer_loop_contiguous.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s

; Test coverage for contiguous access detection in outer loop vectorization.
; Tests various stride and type combinations.

; --- stride-1 float load ---
; for (i = 0; i < N; i++)
;   for (j = 0; j < M; j++)
;     A[i * M + j] *= scale[i];
;
; scale[i] is a stride-1 access w.r.t. the outer loop IV.
define void @scale_rows(ptr noalias %A, ptr noalias %scale, i64 %N, i64 %M) {
; CHECK-LABEL: define void @scale_rows(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[SCALE:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[SCALE]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[J2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i64> [[TMP2]], [[J2]]
; CHECK-NEXT:    [[WIDE_GEP3:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP3]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP3]], <4 x i1> splat (i1 true), <4 x float> poison)
; CHECK-NEXT:    [[TMP4:%.*]] = fmul <4 x float> [[WIDE_MASKED_GATHER]], [[WIDE_LOAD]]
; CHECK-NEXT:    call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> [[TMP4]], <4 x ptr> align 4 [[WIDE_GEP3]], <4 x i1> splat (i1 true))
; CHECK-NEXT:    [[TMP5]] = add nuw nsw <4 x i64> [[J2]], splat (i64 1)
; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0
; CHECK-NEXT:    br i1 [[TMP7]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]]
; CHECK:       [[OUTER_LATCH4]]:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
  %scale.ptr = getelementptr inbounds float, ptr %scale, i64 %i
  %scale.val = load float, ptr %scale.ptr, align 4
  %i.mul.M = mul nsw i64 %i, %M
  br label %inner.body

inner.body:
  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
  %idx = add nsw i64 %i.mul.M, %j
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %mul = fmul float %A.val, %scale.val
  store float %mul, ptr %A.ptr, align 4
  %j.next = add nuw nsw i64 %j, 1
  %j.cmp = icmp eq i64 %j.next, %M
  br i1 %j.cmp, label %outer.latch, label %inner.body

outer.latch:
  %i.next = add nuw nsw i64 %i, 1
  %i.cmp = icmp eq i64 %i.next, %N
  br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; --- stride-2 float load (NOT contiguous) ---
; for (i = 0; i < N; i++)
;   for (j = 0; j < M; j++)
;     A[i*M+j] *= scale[2*i];
;
; scale[2*i] has stride 2*sizeof(float) = 8 bytes, not matching
; sizeof(float) = 4. Should remain a gather.
define void @stride2_float_load(ptr noalias %A, ptr noalias %scale, i64 %N, i64 %M) {
; CHECK-LABEL: define void @stride2_float_load(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[SCALE:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH5:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH5]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[SCALE]], <4 x i64> [[TMP1]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x float> poison)
; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[J2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i64> [[TMP2]], [[J2]]
; CHECK-NEXT:    [[WIDE_GEP3:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP3]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER4:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP3]], <4 x i1> splat (i1 true), <4 x float> poison)
; CHECK-NEXT:    [[TMP4:%.*]] = fmul <4 x float> [[WIDE_MASKED_GATHER4]], [[WIDE_MASKED_GATHER]]
; CHECK-NEXT:    call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> [[TMP4]], <4 x ptr> align 4 [[WIDE_GEP3]], <4 x i1> splat (i1 true))
; CHECK-NEXT:    [[TMP5]] = add nuw nsw <4 x i64> [[J2]], splat (i64 1)
; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0
; CHECK-NEXT:    br i1 [[TMP7]], label %[[OUTER_LATCH5]], label %[[INNER_BODY1]]
; CHECK:       [[OUTER_LATCH5]]:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
  %i2 = shl nuw nsw i64 %i, 1
  %scale.ptr = getelementptr inbounds float, ptr %scale, i64 %i2
  %scale.val = load float, ptr %scale.ptr, align 4
  %i.mul.M = mul nsw i64 %i, %M
  br label %inner.body

inner.body:
  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
  %idx = add nsw i64 %i.mul.M, %j
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %mul = fmul float %A.val, %scale.val
  store float %mul, ptr %A.ptr, align 4
  %j.next = add nuw nsw i64 %j, 1
  %j.cmp = icmp eq i64 %j.next, %M
  br i1 %j.cmp, label %outer.latch, label %inner.body

outer.latch:
  %i.next = add nuw nsw i64 %i, 1
  %i.cmp = icmp eq i64 %i.next, %N
  br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; --- stride-1 double load ---
; for (i = 0; i < N; i++)
;   for (j = 0; j < M; j++)
;     A[i*M+j] *= scale[i];    // double type
;
; scale[i] has stride sizeof(double) = 8, matching the element type.
define void @stride1_double_load(ptr noalias %A, ptr noalias %scale, i64 %N, i64 %M) {
; CHECK-LABEL: define void @stride1_double_load(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[SCALE:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, ptr [[SCALE]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP1]], align 8
; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[J2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i64> [[TMP2]], [[J2]]
; CHECK-NEXT:    [[WIDE_GEP3:%.*]] = getelementptr inbounds double, ptr [[A]], <4 x i64> [[TMP3]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[WIDE_GEP3]], <4 x i1> splat (i1 true), <4 x double> poison)
; CHECK-NEXT:    [[TMP4:%.*]] = fmul <4 x double> [[WIDE_MASKED_GATHER]], [[WIDE_LOAD]]
; CHECK-NEXT:    call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP4]], <4 x ptr> align 8 [[WIDE_GEP3]], <4 x i1> splat (i1 true))
; CHECK-NEXT:    [[TMP5]] = add nuw nsw <4 x i64> [[J2]], splat (i64 1)
; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0
; CHECK-NEXT:    br i1 [[TMP7]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]]
; CHECK:       [[OUTER_LATCH4]]:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
  %scale.ptr = getelementptr inbounds double, ptr %scale, i64 %i
  %scale.val = load double, ptr %scale.ptr, align 8
  %i.mul.M = mul nsw i64 %i, %M
  br label %inner.body

inner.body:
  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
  %idx = add nsw i64 %i.mul.M, %j
  %A.ptr = getelementptr inbounds double, ptr %A, i64 %idx
  %A.val = load double, ptr %A.ptr, align 8
  %mul = fmul double %A.val, %scale.val
  store double %mul, ptr %A.ptr, align 8
  %j.next = add nuw nsw i64 %j, 1
  %j.cmp = icmp eq i64 %j.next, %M
  br i1 %j.cmp, label %outer.latch, label %inner.body

outer.latch:
  %i.next = add nuw nsw i64 %i, 1
  %i.cmp = icmp eq i64 %i.next, %N
  br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; --- stride-1 i1 load and store (NOT contiguous) ---
; for (i = 0; i < N; i++) {
;   bool b = flags[i];
;   for (j = 0; j < M; j++)
;     A[i*M+j] = b;
;   flags[i] = !b;
; }
;
; flags[i] has stride 1 in units of i1, but i1 is bit-packed in vectors: 4
; consecutive i1 scalars span 4 bytes, while a <4 x i1> access covers a single
; byte. Should remain a gather and scatter.
define void @stride1_i1_load_store(ptr noalias %A, ptr noalias %flags, i64 %N, i64 %M) {
; CHECK-LABEL: define void @stride1_i1_load_store(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[FLAGS:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds i1, ptr [[FLAGS]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <4 x i1> @llvm.masked.gather.v4i1.v4p0(<4 x ptr> align 1 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i1> poison)
; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i1> [[WIDE_LOAD]] to <4 x i8>
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[J2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i64> [[TMP2]], [[J2]]
; CHECK-NEXT:    [[WIDE_GEP3:%.*]] = getelementptr inbounds i8, ptr [[A]], <4 x i64> [[TMP4]]
; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP3]], <4 x ptr> align 1 [[WIDE_GEP3]], <4 x i1> splat (i1 true))
; CHECK-NEXT:    [[TMP5]] = add nuw nsw <4 x i64> [[J2]], splat (i64 1)
; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0
; CHECK-NEXT:    br i1 [[TMP7]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]]
; CHECK:       [[OUTER_LATCH4]]:
; CHECK-NEXT:    [[TMP8:%.*]] = xor <4 x i1> [[WIDE_LOAD]], splat (i1 true)
; CHECK-NEXT:    call void @llvm.masked.scatter.v4i1.v4p0(<4 x i1> [[TMP8]], <4 x ptr> align 1 [[WIDE_GEP]], <4 x i1> splat (i1 true))
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
  %flags.ptr = getelementptr inbounds i1, ptr %flags, i64 %i
  %flag = load i1, ptr %flags.ptr, align 1
  %i.mul.M = mul nsw i64 %i, %M
  %flag.ext = zext i1 %flag to i8
  br label %inner.body

inner.body:
  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
  %idx = add nsw i64 %i.mul.M, %j
  %A.ptr = getelementptr inbounds i8, ptr %A, i64 %idx
  store i8 %flag.ext, ptr %A.ptr, align 1
  %j.next = add nuw nsw i64 %j, 1
  %j.cmp = icmp eq i64 %j.next, %M
  br i1 %j.cmp, label %outer.latch, label %inner.body

outer.latch:
  %flag.not = xor i1 %flag, true
  store i1 %flag.not, ptr %flags.ptr, align 1
  %i.next = add nuw nsw i64 %i, 1
  %i.cmp = icmp eq i64 %i.next, %N
  br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; --- stride-1 i4 load and store (NOT contiguous) ---
; Same as above with i4, which is also bit-packed in vectors: 4 consecutive i4
; scalars span 4 bytes, while a <4 x i4> access covers 2 bytes.
define void @stride1_i4_load_store(ptr noalias %A, ptr noalias %vals, i64 %N, i64 %M) {
; CHECK-LABEL: define void @stride1_i4_load_store(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[VALS:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds i4, ptr [[VALS]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <4 x i4> @llvm.masked.gather.v4i4.v4p0(<4 x ptr> align 1 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i4> poison)
; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i4> [[WIDE_LOAD]] to <4 x i8>
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[J2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i64> [[TMP2]], [[J2]]
; CHECK-NEXT:    [[WIDE_GEP3:%.*]] = getelementptr inbounds i8, ptr [[A]], <4 x i64> [[TMP4]]
; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP3]], <4 x ptr> align 1 [[WIDE_GEP3]], <4 x i1> splat (i1 true))
; CHECK-NEXT:    [[TMP5]] = add nuw nsw <4 x i64> [[J2]], splat (i64 1)
; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0
; CHECK-NEXT:    br i1 [[TMP7]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]]
; CHECK:       [[OUTER_LATCH4]]:
; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i4> [[WIDE_LOAD]], splat (i4 1)
; CHECK-NEXT:    call void @llvm.masked.scatter.v4i4.v4p0(<4 x i4> [[TMP8]], <4 x ptr> align 1 [[WIDE_GEP]], <4 x i1> splat (i1 true))
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
  %vals.ptr = getelementptr inbounds i4, ptr %vals, i64 %i
  %val = load i4, ptr %vals.ptr, align 1
  %i.mul.M = mul nsw i64 %i, %M
  %val.ext = zext i4 %val to i8
  br label %inner.body

inner.body:
  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
  %idx = add nsw i64 %i.mul.M, %j
  %A.ptr = getelementptr inbounds i8, ptr %A, i64 %idx
  store i8 %val.ext, ptr %A.ptr, align 1
  %j.next = add nuw nsw i64 %j, 1
  %j.cmp = icmp eq i64 %j.next, %M
  br i1 %j.cmp, label %outer.latch, label %inner.body

outer.latch:
  %val.inc = add i4 %val, 1
  store i4 %val.inc, ptr %vals.ptr, align 1
  %i.next = add nuw nsw i64 %i, 1
  %i.cmp = icmp eq i64 %i.next, %N
  br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; --- stride-1 i8 load and store ---
; Same as above with i8, which is not bit-packed in vectors, so a <4 x i8>
; access covers the same bytes as 4 consecutive i8 scalars.
define void @stride1_i8_load_store(ptr noalias %A, ptr noalias %vals, i64 %N, i64 %M) {
; CHECK-LABEL: define void @stride1_i8_load_store(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[VALS:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[VALS]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[J2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i64> [[TMP2]], [[J2]]
; CHECK-NEXT:    [[WIDE_GEP3:%.*]] = getelementptr inbounds i8, ptr [[A]], <4 x i64> [[TMP3]]
; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[WIDE_LOAD]], <4 x ptr> align 1 [[WIDE_GEP3]], <4 x i1> splat (i1 true))
; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[J2]], splat (i64 1)
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
; CHECK-NEXT:    br i1 [[TMP6]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]]
; CHECK:       [[OUTER_LATCH4]]:
; CHECK-NEXT:    [[TMP7:%.*]] = add <4 x i8> [[WIDE_LOAD]], splat (i8 1)
; CHECK-NEXT:    store <4 x i8> [[TMP7]], ptr [[TMP1]], align 1
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
  %vals.ptr = getelementptr inbounds i8, ptr %vals, i64 %i
  %val = load i8, ptr %vals.ptr, align 1
  %i.mul.M = mul nsw i64 %i, %M
  br label %inner.body

inner.body:
  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
  %idx = add nsw i64 %i.mul.M, %j
  %A.ptr = getelementptr inbounds i8, ptr %A, i64 %idx
  store i8 %val, ptr %A.ptr, align 1
  %j.next = add nuw nsw i64 %j, 1
  %j.cmp = icmp eq i64 %j.next, %M
  br i1 %j.cmp, label %outer.latch, label %inner.body

outer.latch:
  %val.inc = add i8 %val, 1
  store i8 %val.inc, ptr %vals.ptr, align 1
  %i.next = add nuw nsw i64 %i, 1
  %i.cmp = icmp eq i64 %i.next, %N
  br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The column-major access A[i + j*N] is unit-stride with respect to the outer
; loop.
define void @col_major(ptr noalias %A, i64 %N, i64 %M) {
; CHECK-LABEL: define void @col_major(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH5:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH5]] ]
; CHECK-NEXT:    br label %[[INNER_BODY3:.*]]
; CHECK:       [[INNER_BODY3]]:
; CHECK-NEXT:    [[INNER_IV4:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_BODY3]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw <4 x i64> [[INNER_IV4]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i64> [[VEC_IND]], [[TMP1]]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP2]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x float> poison), !llvm.access.group [[ACC_GRP14:![0-9]+]]
; CHECK-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[WIDE_MASKED_GATHER]], splat (float 2.000000e+00)
; CHECK-NEXT:    call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> [[TMP3]], <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true)), !llvm.access.group [[ACC_GRP14]]
; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV4]], splat (i64 1)
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], [[BROADCAST_SPLAT2]]
; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
; CHECK-NEXT:    br i1 [[TMP6]], label %[[OUTER_LATCH5]], label %[[INNER_BODY3]]
; CHECK:       [[OUTER_LATCH5]]:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %inner.iv.mul.N = mul nsw i64 %inner.iv, %N
  %idx = add nsw i64 %outer.iv, %inner.iv.mul.N
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4, !llvm.access.group !3
  %mul = fmul float %A.val, 2.000000e+00
  store float %mul, ptr %A.ptr, align 4, !llvm.access.group !3
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !5

exit:
  ret void
}

; The row-major access A[i*M + j] has stride M with respect to the outer loop.
define void @row_major(ptr noalias %A, i64 %N, i64 %M) {
; CHECK-LABEL: define void @row_major(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH3:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH3]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i64> [[TMP1]], [[INNER_IV2]]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP2]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x float> poison), !llvm.access.group [[ACC_GRP14]]
; CHECK-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[WIDE_MASKED_GATHER]], splat (float 2.000000e+00)
; CHECK-NEXT:    call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> [[TMP3]], <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true)), !llvm.access.group [[ACC_GRP14]]
; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
; CHECK-NEXT:    br i1 [[TMP6]], label %[[OUTER_LATCH3]], label %[[INNER_BODY1]]
; CHECK:       [[OUTER_LATCH3]]:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4, !llvm.access.group !3
  %mul = fmul float %A.val, 2.000000e+00
  store float %mul, ptr %A.ptr, align 4, !llvm.access.group !3
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !5

exit:
  ret void
}

; The inner-loop step of A[i + j*i] varies with the outer loop, so the access
; has no constant stride with respect to the outer loop.
define void @inner_step_varies_with_outer(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @inner_step_varies_with_outer(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH3:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH3]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <4 x i64> [[INNER_IV2]], [[VEC_IND]]
; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i64> [[VEC_IND]], [[TMP2]]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP3]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x float> poison)
; CHECK-NEXT:    [[TMP4:%.*]] = fmul <4 x float> [[WIDE_MASKED_GATHER]], splat (float 2.000000e+00)
; CHECK-NEXT:    store <4 x float> [[TMP4]], ptr [[TMP1]], align 4
; CHECK-NEXT:    [[TMP5]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0
; CHECK-NEXT:    br i1 [[TMP7]], label %[[OUTER_LATCH3]], label %[[INNER_BODY1]]
; CHECK:       [[OUTER_LATCH3]]:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %outer.iv
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %inner.iv.mul.outer.iv = mul nsw i64 %inner.iv, %outer.iv
  %idx = add nsw i64 %outer.iv, %inner.iv.mul.outer.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %mul = fmul float %A.val, 2.000000e+00
  store float %mul, ptr %B.ptr, align 4
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !6

exit:
  ret void
}

; The inner-loop index doubles on each iteration, so the access is not an
; affine recurrence.
define void @inner_step_is_phi(ptr noalias %A, i64 %N, i64 %M) {
; CHECK-LABEL: define void @inner_step_is_phi(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ]
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[INNER_VALUE3:%.*]] = phi <4 x i64> [ splat (i64 1), %[[VECTOR_BODY]] ], [ [[TMP1:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP1]] = add <4 x i64> [[INNER_VALUE3]], [[INNER_VALUE3]]
; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i64> [[INNER_VALUE3]], [[VEC_IND]]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP2]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x float> poison)
; CHECK-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[WIDE_MASKED_GATHER]], splat (float 2.000000e+00)
; CHECK-NEXT:    call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> [[TMP3]], <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true))
; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
; CHECK-NEXT:    br i1 [[TMP6]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]]
; CHECK:       [[OUTER_LATCH4]]:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %inner.value = phi i64 [ 1, %outer.header ], [ %inner.value.next, %inner.body ]
  %inner.value.next = add i64 %inner.value, %inner.value
  %idx = add nsw i64 %inner.value, %outer.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %mul = fmul float %A.val, 2.000000e+00
  store float %mul, ptr %A.ptr, align 4
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !5

exit:
  ret void
}

; Each of the two sibling inner loops accesses A[row*64 + i] with its own
; recurrence row.
define void @col_major_sibling_inner_loops(ptr %A) {
; CHECK-LABEL: define void @col_major_sibling_inner_loops(
; CHECK-SAME: ptr [[A:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH10:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH10]] ]
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP6:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[ROW3:%.*]] = phi <4 x i8> [ splat (i8 2), %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = sext <4 x i8> [[ROW3]] to <4 x i64>
; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> [[TMP0]], splat (i64 6)
; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[TMP1]], [[VEC_IND]]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], <4 x i64> [[TMP2]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison), !llvm.access.group [[ACC_GRP14]]
; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[ROW3]] to <4 x i32>
; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[WIDE_MASKED_GATHER]], [[TMP3]]
; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP4]], <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true)), !llvm.access.group [[ACC_GRP14]]
; CHECK-NEXT:    [[TMP5]] = add <4 x i8> [[ROW3]], splat (i8 1)
; CHECK-NEXT:    [[TMP6]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i64> [[TMP6]], splat (i64 8)
; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
; CHECK-NEXT:    br i1 [[TMP8]], label %[[INNER2_PH4:.*]], label %[[INNER_BODY1]]
; CHECK:       [[INNER2_PH4]]:
; CHECK-NEXT:    br label %[[INNER2_BODY5:.*]]
; CHECK:       [[INNER2_BODY5]]:
; CHECK-NEXT:    [[INNER2_IV6:%.*]] = phi <4 x i64> [ zeroinitializer, %[[INNER2_PH4]] ], [ [[TMP15:%.*]], %[[INNER2_BODY5]] ]
; CHECK-NEXT:    [[ROW27:%.*]] = phi <4 x i8> [ splat (i8 2), %[[INNER2_PH4]] ], [ [[TMP14:%.*]], %[[INNER2_BODY5]] ]
; CHECK-NEXT:    [[TMP9:%.*]] = sext <4 x i8> [[ROW27]] to <4 x i64>
; CHECK-NEXT:    [[TMP10:%.*]] = shl <4 x i64> [[TMP9]], splat (i64 6)
; CHECK-NEXT:    [[TMP11:%.*]] = add <4 x i64> [[TMP10]], [[VEC_IND]]
; CHECK-NEXT:    [[WIDE_GEP8:%.*]] = getelementptr inbounds i32, ptr [[A]], <4 x i64> [[TMP11]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER9:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP8]], <4 x i1> splat (i1 true), <4 x i32> poison), !llvm.access.group [[ACC_GRP14]]
; CHECK-NEXT:    [[TMP12:%.*]] = zext <4 x i8> [[ROW27]] to <4 x i32>
; CHECK-NEXT:    [[TMP13:%.*]] = add <4 x i32> [[WIDE_MASKED_GATHER9]], [[TMP12]]
; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP13]], <4 x ptr> align 4 [[WIDE_GEP8]], <4 x i1> splat (i1 true)), !llvm.access.group [[ACC_GRP14]]
; CHECK-NEXT:    [[TMP14]] = add <4 x i8> [[ROW27]], splat (i8 1)
; CHECK-NEXT:    [[TMP15]] = add nuw nsw <4 x i64> [[INNER2_IV6]], splat (i64 1)
; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq <4 x i64> [[TMP15]], splat (i64 8)
; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i1> [[TMP16]], i64 0
; CHECK-NEXT:    br i1 [[TMP17]], label %[[OUTER_LATCH10]], label %[[INNER2_BODY5]]
; CHECK:       [[OUTER_LATCH10]]:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %row = phi i8 [ 2, %outer.header ], [ %row.next, %inner.body ]
  %row.ext = sext i8 %row to i64
  %row.mul.64 = mul i64 %row.ext, 64
  %idx = add i64 %row.mul.64, %outer.iv
  %A.ptr = getelementptr inbounds i32, ptr %A, i64 %idx
  %A.val = load i32, ptr %A.ptr, align 4, !llvm.access.group !3
  %row.i32 = zext i8 %row to i32
  %add = add i32 %A.val, %row.i32
  store i32 %add, ptr %A.ptr, align 4, !llvm.access.group !3
  %row.next = add i8 %row, 1
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 8
  br i1 %inner.iv.cmp, label %inner2.ph, label %inner.body

inner2.ph:
  br label %inner2.body

inner2.body:
  %inner2.iv = phi i64 [ 0, %inner2.ph ], [ %inner2.iv.next, %inner2.body ]
  %row2 = phi i8 [ 2, %inner2.ph ], [ %row2.next, %inner2.body ]
  %row2.ext = sext i8 %row2 to i64
  %row2.mul.64 = mul i64 %row2.ext, 64
  %idx2 = add i64 %row2.mul.64, %outer.iv
  %A.ptr2 = getelementptr inbounds i32, ptr %A, i64 %idx2
  %A.val2 = load i32, ptr %A.ptr2, align 4, !llvm.access.group !3
  %row2.i32 = zext i8 %row2 to i32
  %add2 = add i32 %A.val2, %row2.i32
  store i32 %add2, ptr %A.ptr2, align 4, !llvm.access.group !3
  %row2.next = add i8 %row2, 1
  %inner2.iv.next = add nuw nsw i64 %inner2.iv, 1
  %inner2.iv.cmp = icmp eq i64 %inner2.iv.next, 8
  br i1 %inner2.iv.cmp, label %outer.latch, label %inner2.body

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, 64
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !5

exit:
  ret void
}

; The recurrence row of A[row*64 + i] is in a loop nested two levels deep.
define void @col_major_nested_inner_loops(ptr %A) {
; CHECK-LABEL: define void @col_major_nested_inner_loops(
; CHECK-SAME: ptr [[A:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH7:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH7]] ]
; CHECK-NEXT:    br label %[[MID_HEADER1:.*]]
; CHECK:       [[MID_HEADER1]]:
; CHECK-NEXT:    [[MID_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP9:%.*]], %[[MID_LATCH6:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY3:.*]]
; CHECK:       [[INNER_BODY3]]:
; CHECK-NEXT:    [[INNER_IV4:%.*]] = phi <4 x i64> [ zeroinitializer, %[[MID_HEADER1]] ], [ [[TMP6:%.*]], %[[INNER_BODY3]] ]
; CHECK-NEXT:    [[ROW5:%.*]] = phi <4 x i8> [ splat (i8 2), %[[MID_HEADER1]] ], [ [[TMP5:%.*]], %[[INNER_BODY3]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = sext <4 x i8> [[ROW5]] to <4 x i64>
; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> [[TMP0]], splat (i64 6)
; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[TMP1]], [[VEC_IND]]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], <4 x i64> [[TMP2]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison), !llvm.access.group [[ACC_GRP14]]
; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[ROW5]] to <4 x i32>
; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[WIDE_MASKED_GATHER]], [[TMP3]]
; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP4]], <4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true)), !llvm.access.group [[ACC_GRP14]]
; CHECK-NEXT:    [[TMP5]] = add <4 x i8> [[ROW5]], splat (i8 1)
; CHECK-NEXT:    [[TMP6]] = add nuw nsw <4 x i64> [[INNER_IV4]], splat (i64 1)
; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i64> [[TMP6]], splat (i64 8)
; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
; CHECK-NEXT:    br i1 [[TMP8]], label %[[MID_LATCH6]], label %[[INNER_BODY3]]
; CHECK:       [[MID_LATCH6]]:
; CHECK-NEXT:    [[TMP9]] = add nuw nsw <4 x i64> [[MID_IV2]], splat (i64 1)
; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <4 x i64> [[TMP9]], splat (i64 2)
; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i64 0
; CHECK-NEXT:    br i1 [[TMP11]], label %[[OUTER_LATCH7]], label %[[MID_HEADER1]]
; CHECK:       [[OUTER_LATCH7]]:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %mid.header

mid.header:
  %mid.iv = phi i64 [ 0, %outer.header ], [ %mid.iv.next, %mid.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %mid.header ], [ %inner.iv.next, %inner.body ]
  %row = phi i8 [ 2, %mid.header ], [ %row.next, %inner.body ]
  %row.ext = sext i8 %row to i64
  %row.mul.64 = mul i64 %row.ext, 64
  %idx = add i64 %row.mul.64, %outer.iv
  %A.ptr = getelementptr inbounds i32, ptr %A, i64 %idx
  %A.val = load i32, ptr %A.ptr, align 4, !llvm.access.group !3
  %row.i32 = zext i8 %row to i32
  %add = add i32 %A.val, %row.i32
  store i32 %add, ptr %A.ptr, align 4, !llvm.access.group !3
  %row.next = add i8 %row, 1
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 8
  br i1 %inner.iv.cmp, label %mid.latch, label %inner.body

mid.latch:
  %mid.iv.next = add nuw nsw i64 %mid.iv, 1
  %mid.iv.cmp = icmp eq i64 %mid.iv.next, 2
  br i1 %mid.iv.cmp, label %outer.latch, label %mid.header

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, 64
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !5

exit:
  ret void
}

!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.vectorize.width", i32 4}
!2 = !{!"llvm.loop.vectorize.enable"}
!3 = distinct !{}
!4 = !{!"llvm.loop.parallel_accesses", !3}
!5 = distinct !{!5, !1, !2, !4}
!6 = distinct !{!6, !1, !2}