test/Transforms/LoopVectorize/expand-scev-cse.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -S %s | FileCheck %s

declare i32 @cond()

; FIXME: ptrtoaddr(%bound) should only be expanded once; see the two
; ptrtoaddr instructions in the CHECK block below.
define i32 @ptrtoaddr_expanded_twice(ptr %bound, ptr %dst) {
; CHECK-LABEL: define i32 @ptrtoaddr_expanded_twice(
; CHECK-SAME: ptr [[BOUND:%.*]], ptr [[DST:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    [[BOUND1:%.*]] = ptrtoaddr ptr [[BOUND]] to i64
; CHECK-NEXT:    br label %[[LOOP1:.*]]
; CHECK:       [[LOOP1]]:
; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[LOOP1_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT:    [[READ:%.*]] = phi ptr [ [[BOUND]], %[[ENTRY]] ], [ [[READ_NEXT:%.*]], %[[LOOP1_LATCH]] ]
; CHECK-NEXT:    [[COND:%.*]] = call i32 @cond()
; CHECK-NEXT:    [[COND_CMP:%.*]] = icmp eq i32 [[COND]], 0
; CHECK-NEXT:    br i1 [[COND_CMP]], label %[[LOOP1_LATCH]], label %[[LOOP2_PREHEADER:.*]]
; CHECK:       [[LOOP2_PREHEADER]]:
; CHECK-NEXT:    [[READ_LCSSA:%.*]] = phi ptr [ [[READ]], %[[LOOP1]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 3, [[BOUND1]]
; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[INDVAR]], -4
; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], [[TMP1]]
; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoaddr ptr [[READ_LCSSA]] to i64
; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoaddr ptr [[BOUND]] to i64
; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP16]], i64 [[TMP15]])
; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[UMAX]], [[TMP3]]
; CHECK-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 2
; CHECK-NEXT:    [[TMP6:%.*]] = add nuw nsw i64 [[TMP5]], 1
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP6]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP6]], 3
; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[TMP7]], 0
; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 4, i64 [[TMP7]]
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP6]], [[TMP9]]
; CHECK-NEXT:    [[TMP10:%.*]] = shl i64 [[N_VEC]], 2
; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[READ_LCSSA]], i64 [[TMP10]]
; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP10]]
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP13:%.*]] = shl i64 [[INDEX]], 2
; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]]
; CHECK-NEXT:    store <4 x i32> splat (i32 1), ptr [[NEXT_GEP]], align 4
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[READ_LCSSA]], %[[LOOP2_PREHEADER]] ]
; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[LOOP2_PREHEADER]] ]
; CHECK-NEXT:    br label %[[LOOP2:.*]]
; CHECK:       [[LOOP1_LATCH]]:
; CHECK-NEXT:    [[READ_NEXT]] = getelementptr i8, ptr [[READ]], i64 4
; CHECK-NEXT:    [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
; CHECK-NEXT:    br label %[[LOOP1]]
; CHECK:       [[LOOP2]]:
; CHECK-NEXT:    [[SCAN:%.*]] = phi ptr [ [[SCAN_NEXT:%.*]], %[[LOOP2_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT:    [[DST_IV:%.*]] = phi ptr [ [[DST_IV_NEXT:%.*]], %[[LOOP2_LATCH]] ], [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ]
; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ult ptr [[SCAN]], [[BOUND]]
; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[LOOP2_LATCH]], label %[[EXIT:.*]]
; CHECK:       [[LOOP2_LATCH]]:
; CHECK-NEXT:    [[SCAN_NEXT]] = getelementptr nusw i8, ptr [[SCAN]], i64 4
; CHECK-NEXT:    [[DST_IV_NEXT]] = getelementptr i8, ptr [[DST_IV]], i64 4
; CHECK-NEXT:    store i32 1, ptr [[DST_IV]], align 4
; CHECK-NEXT:    br label %[[LOOP2]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret i32 0
;
entry:
  br label %loop1

loop1:
  %read = phi ptr [ %bound, %entry ], [ %read.next, %loop1.latch ]
  %cond = call i32 @cond()
  %cond.cmp = icmp eq i32 %cond, 0
  br i1 %cond.cmp, label %loop1.latch, label %loop2

loop1.latch:
  %read.next = getelementptr i8, ptr %read, i64 4
  br label %loop1

loop2:
  %scan = phi ptr [ %scan.next, %loop2.latch ], [ %read, %loop1 ]
  %dst.iv = phi ptr [ %dst.iv.next, %loop2.latch ], [ %dst, %loop1 ]
  %exitcond = icmp ult ptr %scan, %bound
  br i1 %exitcond, label %loop2.latch, label %exit

loop2.latch:
  %scan.next = getelementptr nusw i8, ptr %scan, i64 4
  %dst.iv.next = getelementptr i8, ptr %dst.iv, i64 4
  store i32 1, ptr %dst.iv, align 4
  br label %loop2

exit:
  ret i32 0
}

; FIXME: the second loop's and/sub computing N_VEC should reuse the first
; loop's instead of recomputing it from scratch.
define void @sibling_loops_recompute_min_iters_check(ptr %dst, i64 %n) {
; CHECK-LABEL: define void @sibling_loops_recompute_min_iters_check(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER:.*]]
; CHECK:       [[OUTER]]:
; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ [[IV_OUTER_NEXT:%.*]], %[[OUTER_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw i64 [[IV_OUTER]], -1
; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[IV_OUTER]], 1
; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 [[TMP0]])
; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[SMAX]], [[TMP1]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK2:%.*]] = icmp ult i64 [[TMP2]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK2]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH3:.*]]
; CHECK:       [[VECTOR_PH3]]:
; CHECK-NEXT:    [[TMP3:%.*]] = and i64 [[TMP2]], 3
; CHECK-NEXT:    [[N_VEC4:%.*]] = sub i64 [[TMP2]], [[TMP3]]
; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[IV_OUTER]], [[N_VEC4]]
; CHECK-NEXT:    br label %[[VECTOR_BODY5:.*]]
; CHECK:       [[VECTOR_BODY5]]:
; CHECK-NEXT:    [[INDEX6:%.*]] = phi i64 [ 0, %[[VECTOR_PH3]] ], [ [[INDEX_NEXT7:%.*]], %[[VECTOR_BODY5]] ]
; CHECK-NEXT:    [[TMP5:%.*]] = add nuw i64 [[IV_OUTER]], [[INDEX6]]
; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr float, ptr [[DST]], i64 [[TMP5]]
; CHECK-NEXT:    store <4 x float> splat (float 1.000000e+00), ptr [[TMP6]], align 4
; CHECK-NEXT:    [[INDEX_NEXT7]] = add nuw i64 [[INDEX6]], 4
; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT7]], [[N_VEC4]]
; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK8:.*]], label %[[VECTOR_BODY5]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK8]]:
; CHECK-NEXT:    [[CMP_N9:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC4]]
; CHECK-NEXT:    br i1 [[CMP_N9]], label %[[INNER1_PREHEADER:.*]], label %[[SCALAR_PH1]]
; CHECK:       [[SCALAR_PH1]]:
; CHECK-NEXT:    [[BC_RESUME_VAL10:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK8]] ], [ [[IV_OUTER]], %[[OUTER]] ]
; CHECK-NEXT:    br label %[[INNER0:.*]]
; CHECK:       [[INNER0]]:
; CHECK-NEXT:    [[IV0:%.*]] = phi i64 [ [[IV0_NEXT:%.*]], %[[INNER0]] ], [ [[BC_RESUME_VAL10]], %[[SCALAR_PH1]] ]
; CHECK-NEXT:    [[P0:%.*]] = getelementptr float, ptr [[DST]], i64 [[IV0]]
; CHECK-NEXT:    store float 1.000000e+00, ptr [[P0]], align 4
; CHECK-NEXT:    [[IV0_NEXT]] = add nuw nsw i64 [[IV0]], 1
; CHECK-NEXT:    [[CMP0:%.*]] = icmp slt i64 [[IV0_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[CMP0]], label %[[INNER0]], label %[[INNER1_PREHEADER]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK:       [[INNER1_PREHEADER]]:
; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 [[TMP0]])
; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP13]], [[TMP1]]
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP14]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP14]], [[TMP15]]
; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[IV_OUTER]], [[N_VEC]]
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP10:%.*]] = add nuw i64 [[IV_OUTER]], [[INDEX]]
; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr float, ptr [[DST]], i64 [[TMP10]]
; CHECK-NEXT:    store <4 x float> splat (float 2.000000e+00), ptr [[TMP11]], align 4
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP14]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ [[IV_OUTER]], %[[INNER1_PREHEADER]] ]
; CHECK-NEXT:    br label %[[INNER1:.*]]
; CHECK:       [[INNER1]]:
; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[IV1_NEXT:%.*]], %[[INNER1]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
; CHECK-NEXT:    [[P1:%.*]] = getelementptr float, ptr [[DST]], i64 [[IV1]]
; CHECK-NEXT:    store float 2.000000e+00, ptr [[P1]], align 4
; CHECK-NEXT:    [[IV1_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i64 [[IV1_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[CMP1]], label %[[INNER1]], label %[[OUTER_LATCH]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[OUTER]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer

outer:
  %iv.outer = phi i64 [ %iv.outer.next, %outer.latch ], [ 0, %entry ]
  br label %inner0

inner0:
  %iv0 = phi i64 [ %iv0.next, %inner0 ], [ %iv.outer, %outer ]
  %p0 = getelementptr float, ptr %dst, i64 %iv0
  store float 1.000000e+00, ptr %p0, align 4
  %iv0.next = add nuw nsw i64 %iv0, 1
  %cmp0 = icmp slt i64 %iv0.next, %n
  br i1 %cmp0, label %inner0, label %inner1

inner1:
  %iv1 = phi i64 [ %iv1.next, %inner1 ], [ %iv.outer, %inner0 ]
  %p1 = getelementptr float, ptr %dst, i64 %iv1
  store float 2.000000e+00, ptr %p1, align 4
  %iv1.next = add nuw nsw i64 %iv1, 1
  %cmp1 = icmp slt i64 %iv1.next, %n
  br i1 %cmp1, label %inner1, label %outer.latch

outer.latch:
  %iv.outer.next = add nuw nsw i64 %iv.outer, 1
  %exitcond = icmp eq i64 %iv.outer.next, %n
  br i1 %exitcond, label %exit, label %outer

exit:
  ret void
}

; FIXME: the umax/umin/sub expression computing the inner loop's trip count
; (which depends on the outer AddRec %i) is expanded three times over.
@arr = internal global [8193 x i8] zeroinitializer

define i32 @nested_loop_min_iters_check_expanded_repeatedly(i64 %n) {
; CHECK-LABEL: define i32 @nested_loop_min_iters_check_expanded_repeatedly(
; CHECK-SAME: i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER:.*]]
; CHECK:       [[OUTER]]:
; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[OUTER_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT:    [[COUNT:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[COUNT_NEXT:%.*]], %[[OUTER_LATCH]] ]
; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 2, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH]] ]
; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[INDVAR]], -2
; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], -4
; CHECK-NEXT:    [[OUTER_COND:%.*]] = icmp ne i64 [[I]], [[N]]
; CHECK-NEXT:    br i1 [[OUTER_COND]], label %[[OUTER_BODY:.*]], label %[[EXIT:.*]]
; CHECK:       [[OUTER_BODY]]:
; CHECK-NEXT:    [[PTR_I:%.*]] = getelementptr inbounds nuw i8, ptr @arr, i64 [[I]]
; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr [[PTR_I]], align 1
; CHECK-NEXT:    [[SKIP:%.*]] = icmp eq i8 [[TMP6]], 0
; CHECK-NEXT:    br i1 [[SKIP]], label %[[OUTER_LATCH]], label %[[INNER_PH:.*]]
; CHECK:       [[INNER_PH]]:
; CHECK-NEXT:    [[K_START:%.*]] = shl nuw i64 [[I]], 1
; CHECK-NEXT:    [[TMP28:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 [[K_START]])
; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP28]], [[TMP3]]
; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP4]], i64 1)
; CHECK-NEXT:    [[TMP29:%.*]] = add i64 [[TMP28]], [[TMP3]]
; CHECK-NEXT:    [[TMP30:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP29]], i64 1)
; CHECK-NEXT:    [[TMP31:%.*]] = add i64 [[TMP28]], [[TMP3]]
; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP31]], [[TMP30]]
; CHECK-NEXT:    [[TMP8:%.*]] = udiv i64 [[TMP5]], [[I]]
; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP7]], [[TMP8]]
; CHECK-NEXT:    [[TMP32:%.*]] = add i64 [[TMP9]], 1
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP32]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP10:%.*]] = and i64 [[TMP32]], 3
; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 0
; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 4, i64 [[TMP10]]
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP32]], [[TMP12]]
; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[N_VEC]], [[I]]
; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[K_START]], [[TMP13]]
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[INDEX]], [[I]]
; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[K_START]], [[TMP15]]
; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 1, [[I]]
; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP16]], [[TMP17]]
; CHECK-NEXT:    [[TMP19:%.*]] = mul i64 2, [[I]]
; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[TMP16]], [[TMP19]]
; CHECK-NEXT:    [[TMP21:%.*]] = mul i64 3, [[I]]
; CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[TMP16]], [[TMP21]]
; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr @arr, i64 [[TMP16]]
; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr @arr, i64 [[TMP18]]
; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds nuw i8, ptr @arr, i64 [[TMP20]]
; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr @arr, i64 [[TMP22]]
; CHECK-NEXT:    store i8 0, ptr [[TMP23]], align 1
; CHECK-NEXT:    store i8 0, ptr [[TMP24]], align 1
; CHECK-NEXT:    store i8 0, ptr [[TMP25]], align 1
; CHECK-NEXT:    store i8 0, ptr [[TMP26]], align 1
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP27]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ [[K_START]], %[[INNER_PH]] ]
; CHECK-NEXT:    br label %[[INNER:.*]]
; CHECK:       [[INNER]]:
; CHECK-NEXT:    [[K:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[K_NEXT:%.*]], %[[INNER_BODY:.*]] ]
; CHECK-NEXT:    [[INNER_COND:%.*]] = icmp samesign ult i64 [[K]], [[N]]
; CHECK-NEXT:    br i1 [[INNER_COND]], label %[[INNER_BODY]], label %[[INNER_EXIT:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[PTR_K:%.*]] = getelementptr inbounds nuw i8, ptr @arr, i64 [[K]]
; CHECK-NEXT:    store i8 0, ptr [[PTR_K]], align 1
; CHECK-NEXT:    [[K_NEXT]] = add nuw nsw i64 [[K]], [[I]]
; CHECK-NEXT:    br label %[[INNER]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK:       [[INNER_EXIT]]:
; CHECK-NEXT:    [[COUNT_INC:%.*]] = add nsw i32 [[COUNT]], 1
; CHECK-NEXT:    br label %[[OUTER_LATCH]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[COUNT_NEXT]] = phi i32 [ [[COUNT_INC]], %[[INNER_EXIT]] ], [ [[COUNT]], %[[OUTER_BODY]] ]
; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT:    [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
; CHECK-NEXT:    br label %[[OUTER]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[COUNT_LCSSA:%.*]] = phi i32 [ [[COUNT]], %[[OUTER]] ]
; CHECK-NEXT:    ret i32 [[COUNT_LCSSA]]
;
entry:
  br label %outer

outer:
  %count = phi i32 [ 0, %entry ], [ %count.next, %outer.latch ]
  %i = phi i64 [ 2, %entry ], [ %i.next, %outer.latch ]
  %outer.cond = icmp ne i64 %i, %n
  br i1 %outer.cond, label %outer.body, label %exit

outer.body:
  %ptr.i = getelementptr inbounds nuw i8, ptr @arr, i64 %i
  %0 = load i8, ptr %ptr.i, align 1
  %skip = icmp eq i8 %0, 0
  br i1 %skip, label %outer.latch, label %inner.ph

inner.ph:
  %k.start = shl nuw nsw i64 %i, 1
  br label %inner

inner:
  %k = phi i64 [ %k.start, %inner.ph ], [ %k.next, %inner.body ]
  %inner.cond = icmp samesign ult i64 %k, %n
  br i1 %inner.cond, label %inner.body, label %inner.exit

inner.body:
  %ptr.k = getelementptr inbounds nuw i8, ptr @arr, i64 %k
  store i8 0, ptr %ptr.k, align 1
  %k.next = add nuw nsw i64 %k, %i
  br label %inner

inner.exit:
  %count.inc = add nsw i32 %count, 1
  br label %outer.latch

outer.latch:
  %count.next = phi i32 [ %count.inc, %inner.exit ], [ %count, %outer.body ]
  %i.next = add nuw nsw i64 %i, 1
  br label %outer

exit:
  %count.lcssa = phi i32 [ %count, %outer ]
  ret i32 %count.lcssa
}