llvm/test/Transforms/LoopVectorize/outer-loop-memory-safety.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -vplan-outer-loop-legality-checks -force-vector-width=4 -S %s | FileCheck %s

; The inner loop only reads, and the outer loop stores to a distinct object at a
; unit-stride address.
define void @inner_reads_outer_store(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @inner_reads_outer_store(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[SUM3:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i64> [[TMP1]], [[INNER_IV2]]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP2]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x float> poison)
; CHECK-NEXT:    [[TMP3]] = fadd <4 x float> [[SUM3]], [[WIDE_MASKED_GATHER]]
; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
; CHECK-NEXT:    br i1 [[TMP6]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]]
; CHECK:       [[OUTER_LATCH4]]:
; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT:    store <4 x float> [[TMP3]], ptr [[TMP7]], align 4
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi float [ 0.000000e+00, %outer.header ], [ %sum.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %sum.next = fadd float %sum, %A.val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %outer.iv
  store float %sum.next, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; Each of the two stores writes a distinct object.
define void @two_distinct_stores(ptr noalias %A, ptr noalias %B, ptr noalias %C, i64 %N, i64 %M) {
; CHECK-LABEL: define void @two_distinct_stores(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[SUM3:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i64> [[TMP1]], [[INNER_IV2]]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP2]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x float> poison)
; CHECK-NEXT:    [[TMP3]] = fadd <4 x float> [[SUM3]], [[WIDE_MASKED_GATHER]]
; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
; CHECK-NEXT:    br i1 [[TMP6]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]]
; CHECK:       [[OUTER_LATCH4]]:
; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT:    store <4 x float> [[TMP3]], ptr [[TMP7]], align 4
; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[INDEX]]
; CHECK-NEXT:    store <4 x float> [[TMP3]], ptr [[TMP8]], align 4
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi float [ 0.000000e+00, %outer.header ], [ %sum.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %sum.next = fadd float %sum, %A.val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %outer.iv
  store float %sum.next, ptr %B.ptr, align 4
  %C.ptr = getelementptr inbounds float, ptr %C, i64 %outer.iv
  store float %sum.next, ptr %C.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The store advances by more than the bytes it writes, so the lanes write
; disjoint elements.
define void @store_strided(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @store_strided(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[SUM3:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i64> [[TMP1]], [[INNER_IV2]]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP2]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x float> poison)
; CHECK-NEXT:    [[TMP3]] = fadd <4 x float> [[SUM3]], [[WIDE_MASKED_GATHER]]
; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
; CHECK-NEXT:    br i1 [[TMP6]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]]
; CHECK:       [[OUTER_LATCH4]]:
; CHECK-NEXT:    [[TMP7:%.*]] = shl nsw <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT:    [[WIDE_GEP5:%.*]] = getelementptr inbounds float, ptr [[B]], <4 x i64> [[TMP7]]
; CHECK-NEXT:    call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> [[TMP3]], <4 x ptr> align 4 [[WIDE_GEP5]], <4 x i1> splat (i1 true))
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi float [ 0.000000e+00, %outer.header ], [ %sum.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %sum.next = fadd float %sum, %A.val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.mul.2 = mul nsw i64 %outer.iv, 2
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %outer.iv.mul.2
  store float %sum.next, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The store may alias the load, but the memory both access over the whole nest
; can be bounded.
define void @may_alias_load(ptr %A, ptr %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @may_alias_load(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_M:%.*]] = mul nsw i64 [[OUTER_IV]], [[M]]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[SUM:%.*]] = phi float [ 0.000000e+00, %[[OUTER_HEADER]] ], [ [[SUM_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV_MUL_M]], [[INNER_IV]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[SUM_NEXT]] = fadd float [[SUM]], [[A_VAL]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi float [ [[SUM_NEXT]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    store float [[SUM_NEXT_LCSSA]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi float [ 0.000000e+00, %outer.header ], [ %sum.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %sum.next = fadd float %sum, %A.val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %outer.iv
  store float %sum.next, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The store may alias both loads.
define void @may_alias_two_loads(ptr %A, ptr %B, ptr %C, i64 %N, i64 %M) {
; CHECK-LABEL: define void @may_alias_two_loads(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_M:%.*]] = mul nsw i64 [[OUTER_IV]], [[M]]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[SUM:%.*]] = phi float [ 0.000000e+00, %[[OUTER_HEADER]] ], [ [[SUM_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV_MUL_M]], [[INNER_IV]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[C_PTR:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[IDX]]
; CHECK-NEXT:    [[C_VAL:%.*]] = load float, ptr [[C_PTR]], align 4
; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[A_VAL]], [[C_VAL]]
; CHECK-NEXT:    [[SUM_NEXT]] = fadd float [[SUM]], [[ADD]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi float [ [[SUM_NEXT]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    store float [[SUM_NEXT_LCSSA]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi float [ 0.000000e+00, %outer.header ], [ %sum.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %C.ptr = getelementptr inbounds float, ptr %C, i64 %idx
  %C.val = load float, ptr %C.ptr, align 4
  %add = fadd float %A.val, %C.val
  %sum.next = fadd float %sum, %add
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %outer.iv
  store float %sum.next, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The load may alias the store, and only dereferences the pointer of the last
; inner iteration, so the inbounds GEPs of earlier iterations may be poison.
define void @may_alias_load_after_inner_loop_pos_store_index(ptr %A, ptr %B) {
; CHECK-LABEL: define void @may_alias_load_after_inner_loop_pos_store_index(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[INNER_IV_MUL_SMAX:%.*]] = mul i64 [[INNER_IV]], 9223372036854775807
; CHECK-NEXT:    [[IDX:%.*]] = add i64 [[OUTER_IV]], [[INNER_IV_MUL_SMAX]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 3
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[A_PTR_LCSSA:%.*]] = phi ptr [ [[A_PTR]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[A_VAL:%.*]] = load i8, ptr [[A_PTR_LCSSA]], align 1
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    store i8 [[A_VAL]], ptr [[B_PTR]], align 1
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 4
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %inner.iv.mul.smax = mul i64 %inner.iv, 9223372036854775807
  %idx = add i64 %outer.iv, %inner.iv.mul.smax
  %A.ptr = getelementptr inbounds i8, ptr %A, i64 %idx
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 3
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %A.ptr.lcssa = phi ptr [ %A.ptr, %inner.body ]
  %A.val = load i8, ptr %A.ptr.lcssa, align 1
  %B.ptr = getelementptr inbounds i8, ptr %B, i64 %outer.iv
  store i8 %A.val, ptr %B.ptr, align 1
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, 4
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; Like @may_alias_load_after_inner_loop, but with an decreasing store index.
define void @may_alias_load_after_inner_loop_neg_store_index(ptr %A, ptr %B) {
; CHECK-LABEL: define void @may_alias_load_after_inner_loop_neg_store_index(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[INNER_IV_SCALED:%.*]] = mul i64 [[INNER_IV]], 9223372036854775807
; CHECK-NEXT:    [[IDX:%.*]] = add i64 [[OUTER_IV]], [[INNER_IV_SCALED]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 3
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[A_PTR_LCSSA:%.*]] = phi ptr [ [[A_PTR]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[A_VAL:%.*]] = load i8, ptr [[A_PTR_LCSSA]], align 1
; CHECK-NEXT:    [[NEG_OUTER_IV:%.*]] = sub i64 0, [[OUTER_IV]]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[NEG_OUTER_IV]]
; CHECK-NEXT:    store i8 [[A_VAL]], ptr [[B_PTR]], align 1
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 4
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %inner.iv.mul.smax = mul i64 %inner.iv, 9223372036854775807
  %idx = add i64 %outer.iv, %inner.iv.mul.smax
  %A.ptr = getelementptr inbounds i8, ptr %A, i64 %idx
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 3
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %A.ptr.lcssa = phi ptr [ %A.ptr, %inner.body ]
  %A.val = load i8, ptr %A.ptr.lcssa, align 1
  %outer.iv.neg = sub i64 0, %outer.iv
  %B.ptr = getelementptr inbounds i8, ptr %B, i64 %outer.iv.neg
  store i8 %A.val, ptr %B.ptr, align 1
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, 4
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; Same as @may_alias_load_after_inner_loop, but with noalias arguments.
define void @load_after_inner_loop_pos_store_index(ptr noalias %A, ptr noalias %B) {
; CHECK-LABEL: define void @load_after_inner_loop_pos_store_index(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP2:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = mul <4 x i64> [[INNER_IV2]], splat (i64 9223372036854775807)
; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[TMP0]]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds i8, ptr [[A]], <4 x i64> [[TMP1]]
; CHECK-NEXT:    [[TMP2]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <4 x i64> [[TMP2]], splat (i64 3)
; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
; CHECK-NEXT:    br i1 [[TMP4]], label %[[OUTER_LATCH3:.*]], label %[[INNER_BODY1]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK:       [[OUTER_LATCH3]]:
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i8> poison)
; CHECK-NEXT:    store <4 x i8> [[WIDE_MASKED_GATHER]], ptr [[B]], align 1
; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[EXIT:.*]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %inner.iv.mul.smax = mul i64 %inner.iv, 9223372036854775807
  %idx = add i64 %outer.iv, %inner.iv.mul.smax
  %A.ptr = getelementptr inbounds i8, ptr %A, i64 %idx
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 3
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %A.ptr.lcssa = phi ptr [ %A.ptr, %inner.body ]
  %A.val = load i8, ptr %A.ptr.lcssa, align 1
  %B.ptr = getelementptr inbounds i8, ptr %B, i64 %outer.iv
  store i8 %A.val, ptr %B.ptr, align 1
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, 4
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; Like @load_after_inner_loop_pos_store_index, but with an decreasing store index.
define void @load_after_inner_loop_neg_store_index(ptr noalias %A, ptr noalias %B) {
; CHECK-LABEL: define void @load_after_inner_loop_neg_store_index(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
; CHECK-NEXT:  [[VECTOR_PH:.*]]:
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[OUTER_IV_NEXT:%.*]], %[[MIDDLE_BLOCK:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[INNER_IV_MUL_SMAX:%.*]] = mul i64 [[INNER_IV]], 9223372036854775807
; CHECK-NEXT:    [[IDX:%.*]] = add i64 [[OUTER_IV]], [[INNER_IV_MUL_SMAX]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 3
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[MIDDLE_BLOCK]], label %[[INNER_BODY1]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[A_PTR_LCSSA:%.*]] = phi ptr [ [[A_PTR]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[A_VAL:%.*]] = load i8, ptr [[A_PTR_LCSSA]], align 1
; CHECK-NEXT:    [[OUTER_IV_NEG:%.*]] = sub i64 0, [[OUTER_IV]]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[OUTER_IV_NEG]]
; CHECK-NEXT:    store i8 [[A_VAL]], ptr [[B_PTR]], align 1
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 4
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %inner.iv.mul.smax = mul i64 %inner.iv, 9223372036854775807
  %idx = add i64 %outer.iv, %inner.iv.mul.smax
  %A.ptr = getelementptr inbounds i8, ptr %A, i64 %idx
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 3
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %A.ptr.lcssa = phi ptr [ %A.ptr, %inner.body ]
  %A.val = load i8, ptr %A.ptr.lcssa, align 1
  %outer.iv.neg = sub i64 0, %outer.iv
  %B.ptr = getelementptr inbounds i8, ptr %B, i64 %outer.iv.neg
  store i8 %A.val, ptr %B.ptr, align 1
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, 4
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The load may alias the store, and its address is not formed by an inbounds
; GEP.
define void @may_alias_load_not_inbounds(ptr %A, ptr %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @may_alias_load_not_inbounds(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_M:%.*]] = mul nsw i64 [[OUTER_IV]], [[M]]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[SUM:%.*]] = phi float [ 0.000000e+00, %[[OUTER_HEADER]] ], [ [[SUM_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV_MUL_M]], [[INNER_IV]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr float, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[SUM_NEXT]] = fadd float [[SUM]], [[A_VAL]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi float [ [[SUM_NEXT]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    store float [[SUM_NEXT_LCSSA]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi float [ 0.000000e+00, %outer.header ], [ %sum.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %sum.next = fadd float %sum, %A.val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %outer.iv
  store float %sum.next, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The load and the store may alias, but are in different address spaces.
define void @may_alias_different_address_spaces(ptr addrspace(1) %A, ptr %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @may_alias_different_address_spaces(
; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_M:%.*]] = mul nsw i64 [[OUTER_IV]], [[M]]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[SUM:%.*]] = phi float [ 0.000000e+00, %[[OUTER_HEADER]] ], [ [[SUM_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV_MUL_M]], [[INNER_IV]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr addrspace(1) [[A_PTR]], align 4
; CHECK-NEXT:    [[SUM_NEXT]] = fadd float [[SUM]], [[A_VAL]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi float [ [[SUM_NEXT]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    store float [[SUM_NEXT_LCSSA]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi float [ 0.000000e+00, %outer.header ], [ %sum.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr addrspace(1) %A, i64 %idx
  %A.val = load float, ptr addrspace(1) %A.ptr, align 4
  %sum.next = fadd float %sum, %A.val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %outer.iv
  store float %sum.next, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The load may alias the store, and is only executed conditionally.
define void @may_alias_load_not_executed_every_iteration(ptr %A, ptr %B, i64 %N, i64 %M, i1 %cond) {
; CHECK-LABEL: define void @may_alias_load_not_executed_every_iteration(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]], i1 [[COND:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_M:%.*]] = mul nsw i64 [[OUTER_IV]], [[M]]
; CHECK-NEXT:    br label %[[INNER_HEADER:.*]]
; CHECK:       [[INNER_HEADER]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_LATCH:.*]] ]
; CHECK-NEXT:    [[SUM:%.*]] = phi float [ 0.000000e+00, %[[OUTER_HEADER]] ], [ [[SUM_NEXT:%.*]], %[[INNER_LATCH]] ]
; CHECK-NEXT:    br i1 [[COND]], label %[[INNER_THEN:.*]], label %[[INNER_LATCH]]
; CHECK:       [[INNER_THEN]]:
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV_MUL_M]], [[INNER_IV]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A_PTR]], align 4
; CHECK-NEXT:    br label %[[INNER_LATCH]]
; CHECK:       [[INNER_LATCH]]:
; CHECK-NEXT:    [[VAL:%.*]] = phi float [ [[A_VAL]], %[[INNER_THEN]] ], [ 0.000000e+00, %[[INNER_HEADER]] ]
; CHECK-NEXT:    [[SUM_NEXT]] = fadd float [[SUM]], [[VAL]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_HEADER]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi float [ [[SUM_NEXT]], %[[INNER_LATCH]] ]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    store float [[SUM_NEXT_LCSSA]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.header

inner.header:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.latch ]
  %sum = phi float [ 0.000000e+00, %outer.header ], [ %sum.next, %inner.latch ]
  br i1 %cond, label %inner.then, label %inner.latch

inner.then:
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  br label %inner.latch

inner.latch:
  %val = phi float [ %A.val, %inner.then ], [ 0.000000e+00, %inner.header ]
  %sum.next = fadd float %sum, %val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.header

outer.latch:
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %outer.iv
  store float %sum.next, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The store address does not vary with the outer loop, so all lanes write the
; same location.
define void @store_outer_invariant(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @store_outer_invariant(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_M:%.*]] = mul nsw i64 [[OUTER_IV]], [[M]]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[SUM:%.*]] = phi float [ 0.000000e+00, %[[OUTER_HEADER]] ], [ [[SUM_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV_MUL_M]], [[INNER_IV]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[SUM_NEXT]] = fadd float [[SUM]], [[A_VAL]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi float [ [[SUM_NEXT]], %[[INNER_BODY]] ]
; CHECK-NEXT:    store float [[SUM_NEXT_LCSSA]], ptr [[B]], align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi float [ 0.000000e+00, %outer.header ], [ %sum.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %sum.next = fadd float %sum, %A.val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  store float %sum.next, ptr %B, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The store address is not formed by an inbounds GEP, so it may wrap and two
; outer iterations may write the same location.
define void @store_not_inbounds(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @store_not_inbounds(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_M:%.*]] = mul nsw i64 [[OUTER_IV]], [[M]]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[SUM:%.*]] = phi float [ 0.000000e+00, %[[OUTER_HEADER]] ], [ [[SUM_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV_MUL_M]], [[INNER_IV]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[SUM_NEXT]] = fadd float [[SUM]], [[A_VAL]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi float [ [[SUM_NEXT]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr float, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    store float [[SUM_NEXT_LCSSA]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi float [ 0.000000e+00, %outer.header ], [ %sum.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %sum.next = fadd float %sum, %A.val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr float, ptr %B, i64 %outer.iv
  store float %sum.next, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The inbounds store GEP is not executed in every outer iteration, so it may
; wrap.
define void @store_not_executed_every_iteration(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M, i1 %cond) {
; CHECK-LABEL: define void @store_not_executed_every_iteration(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]], i1 [[COND:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_M:%.*]] = mul nsw i64 [[OUTER_IV]], [[M]]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[SUM:%.*]] = phi float [ 0.000000e+00, %[[OUTER_HEADER]] ], [ [[SUM_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV_MUL_M]], [[INNER_IV]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[SUM_NEXT]] = fadd float [[SUM]], [[A_VAL]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_BODY:.*]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_BODY]]:
; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi float [ [[SUM_NEXT]], %[[INNER_BODY]] ]
; CHECK-NEXT:    br i1 [[COND]], label %[[OUTER_THEN:.*]], label %[[OUTER_LATCH]]
; CHECK:       [[OUTER_THEN]]:
; CHECK-NEXT:    [[OUTER_IV_MUL_2:%.*]] = mul nsw i64 [[OUTER_IV]], 2
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OUTER_IV_MUL_2]]
; CHECK-NEXT:    store float [[SUM_NEXT_LCSSA]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    br label %[[OUTER_LATCH]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi float [ 0.000000e+00, %outer.header ], [ %sum.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %sum.next = fadd float %sum, %A.val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.body, label %inner.body

outer.body:
  br i1 %cond, label %outer.then, label %outer.latch

outer.then:
  %outer.iv.mul.2 = mul nsw i64 %outer.iv, 2
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %outer.iv.mul.2
  store float %sum.next, ptr %B.ptr, align 4
  br label %outer.latch

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; Only the outer GEP is inbounds; the inner one wraps, so lanes 0 and 2 store to
; the same location.
define void @store_chained_gep_not_inbounds(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @store_chained_gep_not_inbounds(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[SUM:%.*]] = phi i8 [ 0, %[[OUTER_HEADER]] ], [ [[SUM_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INNER_IV]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load i8, ptr [[A_PTR]], align 1
; CHECK-NEXT:    [[SUM_NEXT]] = add i8 [[SUM]], [[A_VAL]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i8 [ [[SUM_NEXT]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_SMIN:%.*]] = mul i64 [[OUTER_IV]], -9223372036854775808
; CHECK-NEXT:    [[B_ROW:%.*]] = getelementptr i8, ptr [[B]], i64 [[OUTER_IV_MUL_SMIN]]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds i8, ptr [[B_ROW]], i64 4
; CHECK-NEXT:    store i8 [[SUM_NEXT_LCSSA]], ptr [[B_PTR]], align 1
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi i8 [ 0, %outer.header ], [ %sum.next, %inner.body ]
  %A.ptr = getelementptr inbounds i8, ptr %A, i64 %inner.iv
  %A.val = load i8, ptr %A.ptr, align 1
  %sum.next = add i8 %sum, %A.val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.mul.smin = mul i64 %outer.iv, -9223372036854775808
  %B.row = getelementptr i8, ptr %B, i64 %outer.iv.mul.smin
  %B.ptr = getelementptr inbounds i8, ptr %B.row, i64 4
  store i8 %sum.next, ptr %B.ptr, align 1
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The GEP is nusw, but its index wraps, so lanes 0 and 2 store to the same
; location.
define void @store_nusw_gep_index_wraps(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @store_nusw_gep_index_wraps(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[SUM:%.*]] = phi i8 [ 0, %[[OUTER_HEADER]] ], [ [[SUM_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INNER_IV]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load i8, ptr [[A_PTR]], align 1
; CHECK-NEXT:    [[SUM_NEXT]] = add i8 [[SUM]], [[A_VAL]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i8 [ [[SUM_NEXT]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_SMIN:%.*]] = mul i64 [[OUTER_IV]], -9223372036854775808
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr nusw i8, ptr [[B]], i64 [[OUTER_IV_MUL_SMIN]]
; CHECK-NEXT:    store i8 [[SUM_NEXT_LCSSA]], ptr [[B_PTR]], align 1
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi i8 [ 0, %outer.header ], [ %sum.next, %inner.body ]
  %A.ptr = getelementptr inbounds i8, ptr %A, i64 %inner.iv
  %A.val = load i8, ptr %A.ptr, align 1
  %sum.next = add i8 %sum, %A.val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.mul.smin = mul i64 %outer.iv, -9223372036854775808
  %B.ptr = getelementptr nusw i8, ptr %B, i64 %outer.iv.mul.smin
  store i8 %sum.next, ptr %B.ptr, align 1
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The store advances by 4 bytes but writes 8, so adjacent lanes overlap.
define void @store_wider_than_stride(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @store_wider_than_stride(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_M:%.*]] = mul nsw i64 [[OUTER_IV]], [[M]]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[SUM:%.*]] = phi float [ 0.000000e+00, %[[OUTER_HEADER]] ], [ [[SUM_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV_MUL_M]], [[INNER_IV]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[SUM_NEXT]] = fadd float [[SUM]], [[A_VAL]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi float [ [[SUM_NEXT]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    [[SUM_EXT:%.*]] = fpext float [[SUM_NEXT_LCSSA]] to double
; CHECK-NEXT:    store double [[SUM_EXT]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi float [ 0.000000e+00, %outer.header ], [ %sum.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %sum.next = fadd float %sum, %A.val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %outer.iv
  %sum.ext = fpext float %sum.next to double
  store double %sum.ext, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The inner-loop store writes a distinct row on each outer iteration.
define void @store_in_inner_loop(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @store_in_inner_loop(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_M:%.*]] = mul nsw i64 [[OUTER_IV]], [[M]]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV_MUL_M]], [[INNER_IV]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IDX]]
; CHECK-NEXT:    store float [[A_VAL]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %idx
  store float %A.val, ptr %B.ptr, align 4
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The lanes of the column-major store A[i + j*8] are only distinct for VF <= 8.
define void @col_major_m8(ptr noalias %A, i64 %N) {
; CHECK-LABEL: define void @col_major_m8(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[INNER_IV_MUL_8:%.*]] = mul nsw i64 [[INNER_IV]], 8
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV]], [[INNER_IV_MUL_8]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_FP:%.*]] = sitofp i64 [[OUTER_IV]] to float
; CHECK-NEXT:    [[SUB:%.*]] = fsub float [[OUTER_IV_FP]], [[A_VAL]]
; CHECK-NEXT:    store float [[SUB]], ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 8
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %inner.iv.mul.8 = mul nsw i64 %inner.iv, 8
  %idx = add nsw i64 %outer.iv, %inner.iv.mul.8
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %outer.iv.fp = sitofp i64 %outer.iv to float
  %sub = fsub float %outer.iv.fp, %A.val
  store float %sub, ptr %A.ptr, align 4
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 8
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The lanes of the column-major store A[i + j*2] are only distinct for VF <= 2.
define void @col_major_m2(ptr noalias %A, i64 %N) {
; CHECK-LABEL: define void @col_major_m2(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[INNER_IV_MUL_2:%.*]] = mul nsw i64 [[INNER_IV]], 2
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV]], [[INNER_IV_MUL_2]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_FP:%.*]] = sitofp i64 [[OUTER_IV]] to float
; CHECK-NEXT:    [[SUB:%.*]] = fsub float [[OUTER_IV_FP]], [[A_VAL]]
; CHECK-NEXT:    store float [[SUB]], ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 2
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %inner.iv.mul.2 = mul nsw i64 %inner.iv, 2
  %idx = add nsw i64 %outer.iv, %inner.iv.mul.2
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %outer.iv.fp = sitofp i64 %outer.iv to float
  %sub = fsub float %outer.iv.fp, %A.val
  store float %sub, ptr %A.ptr, align 4
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 2
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; Like @col_major_m2, but without the load, so only the store conflicts.
define void @col_major_m2_store_only(ptr noalias %A, i64 %N) {
; CHECK-LABEL: define void @col_major_m2_store_only(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[INNER_IV_MUL_2:%.*]] = mul nsw i64 [[INNER_IV]], 2
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV]], [[INNER_IV_MUL_2]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    store i32 0, ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 2
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %inner.iv.mul.2 = mul nsw i64 %inner.iv, 2
  %idx = add nsw i64 %outer.iv, %inner.iv.mul.2
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  store i32 0, ptr %A.ptr, align 4
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 2
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The row-major store A[i*8 + j] writes a distinct row on each outer iteration.
define void @row_major_m8(ptr noalias %A, i64 %N) {
; CHECK-LABEL: define void @row_major_m8(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_8:%.*]] = mul nsw i64 [[OUTER_IV]], 8
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV_MUL_8]], [[INNER_IV]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_FP:%.*]] = sitofp i64 [[OUTER_IV]] to float
; CHECK-NEXT:    [[SUB:%.*]] = fsub float [[OUTER_IV_FP]], [[A_VAL]]
; CHECK-NEXT:    store float [[SUB]], ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 8
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.8 = mul nsw i64 %outer.iv, 8
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.8, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %outer.iv.fp = sitofp i64 %outer.iv to float
  %sub = fsub float %outer.iv.fp, %A.val
  store float %sub, ptr %A.ptr, align 4
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 8
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The 4-byte accesses to A[i + j*4] advance by one byte per outer iteration, so
; the lanes overlap.
define void @access_wider_than_outer_stride(ptr noalias %A, i64 %N) {
; CHECK-LABEL: define void @access_wider_than_outer_stride(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[INNER_IV_MUL_4:%.*]] = mul nsw i64 [[INNER_IV]], 4
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV]], [[INNER_IV_MUL_4]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load float, ptr [[A_PTR]], align 1
; CHECK-NEXT:    [[OUTER_IV_FP:%.*]] = sitofp i64 [[OUTER_IV]] to float
; CHECK-NEXT:    [[SUB:%.*]] = fsub float [[OUTER_IV_FP]], [[A_VAL]]
; CHECK-NEXT:    store float [[SUB]], ptr [[A_PTR]], align 1
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 4
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %inner.iv.mul.4 = mul nsw i64 %inner.iv, 4
  %idx = add nsw i64 %outer.iv, %inner.iv.mul.4
  %A.ptr = getelementptr inbounds i8, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 1
  %outer.iv.fp = sitofp i64 %outer.iv to float
  %sub = fsub float %outer.iv.fp, %A.val
  store float %sub, ptr %A.ptr, align 1
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 4
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The volatile load cannot be widened.
define void @volatile_load(ptr noalias %A, ptr noalias %B) {
; CHECK-LABEL: define void @volatile_load(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[A_VAL:%.*]] = load volatile i32, ptr [[A]], align 4
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 2
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[A_VAL_LCSSA:%.*]] = phi i32 [ [[A_VAL]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    store i32 [[A_VAL_LCSSA]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 4
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %A.val = load volatile i32, ptr %A, align 4
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 2
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr inbounds i32, ptr %B, i64 %outer.iv
  store i32 %A.val, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, 4
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The volatile store cannot be widened.
define void @volatile_store(ptr noalias %A, ptr noalias %B) {
; CHECK-LABEL: define void @volatile_store(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[A_VAL:%.*]] = load i32, ptr [[A]], align 4
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 2
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[A_VAL_LCSSA:%.*]] = phi i32 [ [[A_VAL]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    store volatile i32 [[A_VAL_LCSSA]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 4
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %A.val = load i32, ptr %A, align 4
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 2
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr inbounds i32, ptr %B, i64 %outer.iv
  store volatile i32 %A.val, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, 4
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The unordered atomic load cannot be widened.
define void @atomic_load_unordered(ptr noalias %A, ptr noalias %B) {
; CHECK-LABEL: define void @atomic_load_unordered(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[A_VAL:%.*]] = load atomic i32, ptr [[A]] unordered, align 4
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 2
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[A_VAL_LCSSA:%.*]] = phi i32 [ [[A_VAL]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    store i32 [[A_VAL_LCSSA]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 4
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %A.val = load atomic i32, ptr %A unordered, align 4
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 2
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr inbounds i32, ptr %B, i64 %outer.iv
  store i32 %A.val, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, 4
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The monotonic atomic load cannot be widened.
define void @atomic_load_monotonic(ptr noalias %A, ptr noalias %B) {
; CHECK-LABEL: define void @atomic_load_monotonic(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[A_VAL:%.*]] = load atomic i32, ptr [[A]] monotonic, align 4
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 2
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[A_VAL_LCSSA:%.*]] = phi i32 [ [[A_VAL]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    store i32 [[A_VAL_LCSSA]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 4
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %A.val = load atomic i32, ptr %A monotonic, align 4
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 2
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr inbounds i32, ptr %B, i64 %outer.iv
  store i32 %A.val, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, 4
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The unordered atomic store cannot be widened.
define void @atomic_store_unordered(ptr noalias %A, ptr noalias %B) {
; CHECK-LABEL: define void @atomic_store_unordered(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[A_VAL:%.*]] = load i32, ptr [[A]], align 4
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 2
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[A_VAL_LCSSA:%.*]] = phi i32 [ [[A_VAL]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    store atomic i32 [[A_VAL_LCSSA]], ptr [[B_PTR]] unordered, align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 4
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %A.val = load i32, ptr %A, align 4
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, 2
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr inbounds i32, ptr %B, i64 %outer.iv
  store atomic i32 %A.val, ptr %B.ptr unordered, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, 4
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The nest copies between distinct objects using a memory intrinsic.
define void @memcpy_in_nest(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @memcpy_in_nest(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_M:%.*]] = mul nsw i64 [[OUTER_IV]], [[M]]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[OUTER_IV_MUL_M]]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OUTER_IV]]
; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[B_PTR]], ptr [[A_PTR]], i64 4, i1 false)
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %outer.iv.mul.M
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %outer.iv
  call void @llvm.memcpy.p0.p0.i64(ptr %B.ptr, ptr %A.ptr, i64 4, i1 false)
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The store in the inner loop writes B[i], so each outer iteration writes a
; distinct element.
define void @store_in_inner_loop_outer_addr(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @store_in_inner_loop_outer_addr(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH3:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH3]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    br label %[[INNER_BODY1:.*]]
; CHECK:       [[INNER_BODY1]]:
; CHECK-NEXT:    [[INNER_IV2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_BODY1]] ]
; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i64> [[TMP1]], [[INNER_IV2]]
; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP2]]
; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x float> poison)
; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT:    store <4 x float> [[WIDE_MASKED_GATHER]], ptr [[TMP3]], align 4
; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[INNER_IV2]], splat (i64 1)
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
; CHECK-NEXT:    br i1 [[TMP6]], label %[[OUTER_LATCH3]], label %[[INNER_BODY1]]
; CHECK:       [[OUTER_LATCH3]]:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
  %A.val = load float, ptr %A.ptr, align 4
  %B.ptr = getelementptr inbounds float, ptr %B, i64 %outer.iv
  store float %A.val, ptr %B.ptr, align 4
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

; The middle loop is vectorized, and the store address only varies with the
; outermost loop, so all lanes write the same location.
define void @store_varies_only_in_enclosing_loop(ptr noalias %B, i64 %N, i64 %K, i64 %M) {
; CHECK-LABEL: define void @store_varies_only_in_enclosing_loop(
; CHECK-SAME: ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[K:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTERMOST_HEADER:.*]]
; CHECK:       [[OUTERMOST_HEADER]]:
; CHECK-NEXT:    [[OUTERMOST_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTERMOST_IV_NEXT:%.*]], %[[OUTERMOST_LATCH:.*]] ]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OUTERMOST_IV]]
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[OUTERMOST_HEADER]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    store i64 [[OUTER_IV]], ptr [[B_PTR]], align 8
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[OUTERMOST_LATCH]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[OUTERMOST_LATCH]]:
; CHECK-NEXT:    [[OUTERMOST_IV_NEXT]] = add nuw nsw i64 [[OUTERMOST_IV]], 1
; CHECK-NEXT:    [[OUTERMOST_IV_CMP:%.*]] = icmp eq i64 [[OUTERMOST_IV_NEXT]], [[K]]
; CHECK-NEXT:    br i1 [[OUTERMOST_IV_CMP]], label %[[EXIT:.*]], label %[[OUTERMOST_HEADER]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outermost.header

outermost.header:
  %outermost.iv = phi i64 [ 0, %entry ], [ %outermost.iv.next, %outermost.latch ]
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %outermost.header ], [ %outer.iv.next, %outer.latch ]
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %B.ptr = getelementptr inbounds i64, ptr %B, i64 %outermost.iv
  store i64 %outer.iv, ptr %B.ptr, align 8
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %outermost.latch, label %outer.header, !llvm.loop !0

outermost.latch:
  %outermost.iv.next = add nuw nsw i64 %outermost.iv, 1
  %outermost.iv.cmp = icmp eq i64 %outermost.iv.next, %K
  br i1 %outermost.iv.cmp, label %exit, label %outermost.header

exit:
  ret void
}

; The store moves backwards by 4 bytes per outer iteration but writes 8, so
; adjacent lanes overlap.
define void @store_negative_step_narrower_than_store(ptr noalias %A, ptr noalias %B, i64 %N, i64 %M) {
; CHECK-LABEL: define void @store_negative_step_narrower_than_store(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OUTER_IV_MUL_M:%.*]] = mul nsw i64 [[OUTER_IV]], [[M]]
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[SUM:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[SUM_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[OUTER_IV_MUL_M]], [[INNER_IV]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[A_VAL:%.*]] = load i64, ptr [[A_PTR]], align 8
; CHECK-NEXT:    [[SUM_NEXT]] = add i64 [[SUM]], [[A_VAL]]
; CHECK-NEXT:    [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
; CHECK-NEXT:    [[INNER_IV_CMP:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[INNER_IV_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i64 [ [[SUM_NEXT]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[NEG_OUTER_IV:%.*]] = sub nsw i64 0, [[OUTER_IV]]
; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[NEG_OUTER_IV]]
; CHECK-NEXT:    store i64 [[SUM_NEXT_LCSSA]], ptr [[B_PTR]], align 4
; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
; CHECK-NEXT:    [[OUTER_IV_CMP:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_IV_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
  %outer.iv.mul.M = mul nsw i64 %outer.iv, %M
  br label %inner.body

inner.body:
  %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.body ]
  %sum = phi i64 [ 0, %outer.header ], [ %sum.next, %inner.body ]
  %idx = add nsw i64 %outer.iv.mul.M, %inner.iv
  %A.ptr = getelementptr inbounds i64, ptr %A, i64 %idx
  %A.val = load i64, ptr %A.ptr, align 8
  %sum.next = add i64 %sum, %A.val
  %inner.iv.next = add nuw nsw i64 %inner.iv, 1
  %inner.iv.cmp = icmp eq i64 %inner.iv.next, %M
  br i1 %inner.iv.cmp, label %outer.latch, label %inner.body

outer.latch:
  %neg.outer.iv = sub nsw i64 0, %outer.iv
  %B.ptr = getelementptr inbounds i32, ptr %B, i64 %neg.outer.iv
  store i64 %sum.next, ptr %B.ptr, align 4
  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
  %outer.iv.cmp = icmp eq i64 %outer.iv.next, %N
  br i1 %outer.iv.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)

!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.vectorize.enable"}