llvm/test/Transforms/LoopVectorize/expand-scev-shared-subexpressions.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -S %s | FileCheck %s

define void @chained_multi_exit_loops(ptr %dst, i64 %n, i64 %w) {
; CHECK-LABEL: define void @chained_multi_exit_loops(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[W:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP_0:.*]]
; CHECK:       [[LOOP_0]]:
; CHECK-NEXT:    [[J_0:%.*]] = phi i64 [ [[W]], %[[ENTRY]] ], [ [[J_NEXT_0:%.*]], %[[LATCH_0:.*]] ]
; CHECK-NEXT:    [[I_0:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT_0:%.*]], %[[LATCH_0]] ]
; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i64 [[J_0]], [[N]]
; CHECK-NEXT:    br i1 [[C_0]], label %[[LATCH_0]], label %[[EARLY_EXIT_LOOPEXIT11:.*]]
; CHECK:       [[LATCH_0]]:
; CHECK-NEXT:    [[I_NEXT_0]] = add nuw i64 [[I_0]], 1
; CHECK-NEXT:    [[J_NEXT_0]] = add nuw i64 [[J_0]], 1
; CHECK-NEXT:    [[EC_0:%.*]] = icmp eq i64 [[I_NEXT_0]], [[W]]
; CHECK-NEXT:    br i1 [[EC_0]], label %[[LOOP_0_EXIT:.*]], label %[[LOOP_0]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       [[LOOP_0_EXIT]]:
; CHECK-NEXT:    [[J_NEXT_0_LCSSA:%.*]] = phi i64 [ [[J_NEXT_0]], %[[LATCH_0]] ]
; CHECK-NEXT:    br label %[[LOOP_1:.*]]
; CHECK:       [[LOOP_1]]:
; CHECK-NEXT:    [[J_1:%.*]] = phi i64 [ [[J_NEXT_0_LCSSA]], %[[LOOP_0_EXIT]] ], [ [[J_NEXT_1:%.*]], %[[LATCH_1:.*]] ]
; CHECK-NEXT:    [[I_1:%.*]] = phi i64 [ 0, %[[LOOP_0_EXIT]] ], [ [[I_NEXT_1:%.*]], %[[LATCH_1]] ]
; CHECK-NEXT:    [[C_1:%.*]] = icmp ult i64 [[J_1]], [[N]]
; CHECK-NEXT:    br i1 [[C_1]], label %[[LATCH_1]], label %[[EARLY_EXIT_LOOPEXIT10:.*]]
; CHECK:       [[LATCH_1]]:
; CHECK-NEXT:    [[I_NEXT_1]] = add nuw i64 [[I_1]], 1
; CHECK-NEXT:    [[J_NEXT_1]] = add nuw i64 [[J_1]], 1
; CHECK-NEXT:    [[EC_1:%.*]] = icmp eq i64 [[I_NEXT_1]], [[W]]
; CHECK-NEXT:    br i1 [[EC_1]], label %[[LOOP_1_EXIT:.*]], label %[[LOOP_1]], !llvm.loop [[LOOP0]]
; CHECK:       [[LOOP_1_EXIT]]:
; CHECK-NEXT:    [[J_NEXT_1_LCSSA:%.*]] = phi i64 [ [[J_NEXT_1]], %[[LATCH_1]] ]
; CHECK-NEXT:    br label %[[LOOP_2:.*]]
; CHECK:       [[LOOP_2]]:
; CHECK-NEXT:    [[J_2:%.*]] = phi i64 [ [[J_NEXT_1_LCSSA]], %[[LOOP_1_EXIT]] ], [ [[J_NEXT_2:%.*]], %[[LATCH_2:.*]] ]
; CHECK-NEXT:    [[I_2:%.*]] = phi i64 [ 0, %[[LOOP_1_EXIT]] ], [ [[I_NEXT_2:%.*]], %[[LATCH_2]] ]
; CHECK-NEXT:    [[C_2:%.*]] = icmp ult i64 [[J_2]], [[N]]
; CHECK-NEXT:    br i1 [[C_2]], label %[[LATCH_2]], label %[[EARLY_EXIT_LOOPEXIT9:.*]]
; CHECK:       [[LATCH_2]]:
; CHECK-NEXT:    [[I_NEXT_2]] = add nuw i64 [[I_2]], 1
; CHECK-NEXT:    [[J_NEXT_2]] = add nuw i64 [[J_2]], 1
; CHECK-NEXT:    [[EC_2:%.*]] = icmp eq i64 [[I_NEXT_2]], [[W]]
; CHECK-NEXT:    br i1 [[EC_2]], label %[[LOOP_2_EXIT:.*]], label %[[LOOP_2]], !llvm.loop [[LOOP0]]
; CHECK:       [[LOOP_2_EXIT]]:
; CHECK-NEXT:    [[J_NEXT_2_LCSSA:%.*]] = phi i64 [ [[J_NEXT_2]], %[[LATCH_2]] ]
; CHECK-NEXT:    br label %[[LOOP_3:.*]]
; CHECK:       [[LOOP_3]]:
; CHECK-NEXT:    [[J_3:%.*]] = phi i64 [ [[J_NEXT_2_LCSSA]], %[[LOOP_2_EXIT]] ], [ [[J_NEXT_3:%.*]], %[[LATCH_3:.*]] ]
; CHECK-NEXT:    [[I_3:%.*]] = phi i64 [ 0, %[[LOOP_2_EXIT]] ], [ [[I_NEXT_3:%.*]], %[[LATCH_3]] ]
; CHECK-NEXT:    [[C_3:%.*]] = icmp ult i64 [[J_3]], [[N]]
; CHECK-NEXT:    br i1 [[C_3]], label %[[LATCH_3]], label %[[EARLY_EXIT_LOOPEXIT8:.*]]
; CHECK:       [[LATCH_3]]:
; CHECK-NEXT:    [[I_NEXT_3]] = add nuw i64 [[I_3]], 1
; CHECK-NEXT:    [[J_NEXT_3]] = add nuw i64 [[J_3]], 1
; CHECK-NEXT:    [[EC_3:%.*]] = icmp eq i64 [[I_NEXT_3]], [[W]]
; CHECK-NEXT:    br i1 [[EC_3]], label %[[LOOP_3_EXIT:.*]], label %[[LOOP_3]], !llvm.loop [[LOOP0]]
; CHECK:       [[LOOP_3_EXIT]]:
; CHECK-NEXT:    [[J_NEXT_3_LCSSA:%.*]] = phi i64 [ [[J_NEXT_3]], %[[LATCH_3]] ]
; CHECK-NEXT:    br label %[[LOOP_4:.*]]
; CHECK:       [[LOOP_4]]:
; CHECK-NEXT:    [[J_4:%.*]] = phi i64 [ [[J_NEXT_3_LCSSA]], %[[LOOP_3_EXIT]] ], [ [[J_NEXT_4:%.*]], %[[LATCH_4:.*]] ]
; CHECK-NEXT:    [[I_4:%.*]] = phi i64 [ 0, %[[LOOP_3_EXIT]] ], [ [[I_NEXT_4:%.*]], %[[LATCH_4]] ]
; CHECK-NEXT:    [[C_4:%.*]] = icmp ult i64 [[J_4]], [[N]]
; CHECK-NEXT:    br i1 [[C_4]], label %[[LATCH_4]], label %[[EARLY_EXIT_LOOPEXIT7:.*]]
; CHECK:       [[LATCH_4]]:
; CHECK-NEXT:    [[I_NEXT_4]] = add nuw i64 [[I_4]], 1
; CHECK-NEXT:    [[J_NEXT_4]] = add nuw i64 [[J_4]], 1
; CHECK-NEXT:    [[EC_4:%.*]] = icmp eq i64 [[I_NEXT_4]], [[W]]
; CHECK-NEXT:    br i1 [[EC_4]], label %[[LOOP_4_EXIT:.*]], label %[[LOOP_4]], !llvm.loop [[LOOP0]]
; CHECK:       [[LOOP_4_EXIT]]:
; CHECK-NEXT:    [[J_NEXT_4_LCSSA:%.*]] = phi i64 [ [[J_NEXT_4]], %[[LATCH_4]] ]
; CHECK-NEXT:    br label %[[LOOP_5:.*]]
; CHECK:       [[LOOP_5]]:
; CHECK-NEXT:    [[J_5:%.*]] = phi i64 [ [[J_NEXT_4_LCSSA]], %[[LOOP_4_EXIT]] ], [ [[J_NEXT_5:%.*]], %[[LATCH_5:.*]] ]
; CHECK-NEXT:    [[I_5:%.*]] = phi i64 [ 0, %[[LOOP_4_EXIT]] ], [ [[I_NEXT_5:%.*]], %[[LATCH_5]] ]
; CHECK-NEXT:    [[C_5:%.*]] = icmp ult i64 [[J_5]], [[N]]
; CHECK-NEXT:    br i1 [[C_5]], label %[[LATCH_5]], label %[[EARLY_EXIT_LOOPEXIT6:.*]]
; CHECK:       [[LATCH_5]]:
; CHECK-NEXT:    [[I_NEXT_5]] = add nuw i64 [[I_5]], 1
; CHECK-NEXT:    [[J_NEXT_5]] = add nuw i64 [[J_5]], 1
; CHECK-NEXT:    [[EC_5:%.*]] = icmp eq i64 [[I_NEXT_5]], [[W]]
; CHECK-NEXT:    br i1 [[EC_5]], label %[[LOOP_5_EXIT:.*]], label %[[LOOP_5]], !llvm.loop [[LOOP0]]
; CHECK:       [[LOOP_5_EXIT]]:
; CHECK-NEXT:    [[J_NEXT_5_LCSSA:%.*]] = phi i64 [ [[J_NEXT_5]], %[[LATCH_5]] ]
; CHECK-NEXT:    br label %[[LOOP_6:.*]]
; CHECK:       [[LOOP_6]]:
; CHECK-NEXT:    [[J_6:%.*]] = phi i64 [ [[J_NEXT_5_LCSSA]], %[[LOOP_5_EXIT]] ], [ [[J_NEXT_6:%.*]], %[[LATCH_6:.*]] ]
; CHECK-NEXT:    [[I_6:%.*]] = phi i64 [ 0, %[[LOOP_5_EXIT]] ], [ [[I_NEXT_6:%.*]], %[[LATCH_6]] ]
; CHECK-NEXT:    [[C_6:%.*]] = icmp ult i64 [[J_6]], [[N]]
; CHECK-NEXT:    br i1 [[C_6]], label %[[LATCH_6]], label %[[EARLY_EXIT_LOOPEXIT5:.*]]
; CHECK:       [[LATCH_6]]:
; CHECK-NEXT:    [[I_NEXT_6]] = add nuw i64 [[I_6]], 1
; CHECK-NEXT:    [[J_NEXT_6]] = add nuw i64 [[J_6]], 1
; CHECK-NEXT:    [[EC_6:%.*]] = icmp eq i64 [[I_NEXT_6]], [[W]]
; CHECK-NEXT:    br i1 [[EC_6]], label %[[LOOP_6_EXIT:.*]], label %[[LOOP_6]], !llvm.loop [[LOOP0]]
; CHECK:       [[LOOP_6_EXIT]]:
; CHECK-NEXT:    [[J_NEXT_6_LCSSA:%.*]] = phi i64 [ [[J_NEXT_6]], %[[LATCH_6]] ]
; CHECK-NEXT:    br label %[[LOOP_7:.*]]
; CHECK:       [[LOOP_7]]:
; CHECK-NEXT:    [[J_7:%.*]] = phi i64 [ [[J_NEXT_6_LCSSA]], %[[LOOP_6_EXIT]] ], [ [[J_NEXT_7:%.*]], %[[LATCH_7:.*]] ]
; CHECK-NEXT:    [[I_7:%.*]] = phi i64 [ 0, %[[LOOP_6_EXIT]] ], [ [[I_NEXT_7:%.*]], %[[LATCH_7]] ]
; CHECK-NEXT:    [[C_7:%.*]] = icmp ult i64 [[J_7]], [[N]]
; CHECK-NEXT:    br i1 [[C_7]], label %[[LATCH_7]], label %[[EARLY_EXIT_LOOPEXIT4:.*]]
; CHECK:       [[LATCH_7]]:
; CHECK-NEXT:    [[I_NEXT_7]] = add nuw i64 [[I_7]], 1
; CHECK-NEXT:    [[J_NEXT_7]] = add nuw i64 [[J_7]], 1
; CHECK-NEXT:    [[EC_7:%.*]] = icmp eq i64 [[I_NEXT_7]], [[W]]
; CHECK-NEXT:    br i1 [[EC_7]], label %[[LOOP_7_EXIT:.*]], label %[[LOOP_7]], !llvm.loop [[LOOP0]]
; CHECK:       [[LOOP_7_EXIT]]:
; CHECK-NEXT:    [[J_NEXT_7_LCSSA:%.*]] = phi i64 [ [[J_NEXT_7]], %[[LATCH_7]] ]
; CHECK-NEXT:    br label %[[LOOP_8:.*]]
; CHECK:       [[LOOP_8]]:
; CHECK-NEXT:    [[J_8:%.*]] = phi i64 [ [[J_NEXT_7_LCSSA]], %[[LOOP_7_EXIT]] ], [ [[J_NEXT_8:%.*]], %[[LATCH_8:.*]] ]
; CHECK-NEXT:    [[I_8:%.*]] = phi i64 [ 0, %[[LOOP_7_EXIT]] ], [ [[I_NEXT_8:%.*]], %[[LATCH_8]] ]
; CHECK-NEXT:    [[C_8:%.*]] = icmp ult i64 [[J_8]], [[N]]
; CHECK-NEXT:    br i1 [[C_8]], label %[[LATCH_8]], label %[[EARLY_EXIT_LOOPEXIT3:.*]]
; CHECK:       [[LATCH_8]]:
; CHECK-NEXT:    [[I_NEXT_8]] = add nuw i64 [[I_8]], 1
; CHECK-NEXT:    [[J_NEXT_8]] = add nuw i64 [[J_8]], 1
; CHECK-NEXT:    [[EC_8:%.*]] = icmp eq i64 [[I_NEXT_8]], [[W]]
; CHECK-NEXT:    br i1 [[EC_8]], label %[[LOOP_8_EXIT:.*]], label %[[LOOP_8]], !llvm.loop [[LOOP0]]
; CHECK:       [[LOOP_8_EXIT]]:
; CHECK-NEXT:    [[J_NEXT_8_LCSSA:%.*]] = phi i64 [ [[J_NEXT_8]], %[[LATCH_8]] ]
; CHECK-NEXT:    br label %[[LOOP_9:.*]]
; CHECK:       [[LOOP_9]]:
; CHECK-NEXT:    [[J_9:%.*]] = phi i64 [ [[J_NEXT_8_LCSSA]], %[[LOOP_8_EXIT]] ], [ [[J_NEXT_9:%.*]], %[[LATCH_9:.*]] ]
; CHECK-NEXT:    [[I_9:%.*]] = phi i64 [ 0, %[[LOOP_8_EXIT]] ], [ [[I_NEXT_9:%.*]], %[[LATCH_9]] ]
; CHECK-NEXT:    [[C_9:%.*]] = icmp ult i64 [[J_9]], [[N]]
; CHECK-NEXT:    br i1 [[C_9]], label %[[LATCH_9]], label %[[EARLY_EXIT_LOOPEXIT2:.*]]
; CHECK:       [[LATCH_9]]:
; CHECK-NEXT:    [[I_NEXT_9]] = add nuw i64 [[I_9]], 1
; CHECK-NEXT:    [[J_NEXT_9]] = add nuw i64 [[J_9]], 1
; CHECK-NEXT:    [[EC_9:%.*]] = icmp eq i64 [[I_NEXT_9]], [[W]]
; CHECK-NEXT:    br i1 [[EC_9]], label %[[LOOP_9_EXIT:.*]], label %[[LOOP_9]], !llvm.loop [[LOOP0]]
; CHECK:       [[LOOP_9_EXIT]]:
; CHECK-NEXT:    [[J_NEXT_9_LCSSA:%.*]] = phi i64 [ [[J_NEXT_9]], %[[LATCH_9]] ]
; CHECK-NEXT:    br label %[[LOOP_10:.*]]
; CHECK:       [[LOOP_10]]:
; CHECK-NEXT:    [[J_10:%.*]] = phi i64 [ [[J_NEXT_9_LCSSA]], %[[LOOP_9_EXIT]] ], [ [[J_NEXT_10:%.*]], %[[LATCH_10:.*]] ]
; CHECK-NEXT:    [[I_10:%.*]] = phi i64 [ 0, %[[LOOP_9_EXIT]] ], [ [[I_NEXT_10:%.*]], %[[LATCH_10]] ]
; CHECK-NEXT:    [[C_10:%.*]] = icmp ult i64 [[J_10]], [[N]]
; CHECK-NEXT:    br i1 [[C_10]], label %[[LATCH_10]], label %[[EARLY_EXIT_LOOPEXIT1:.*]]
; CHECK:       [[LATCH_10]]:
; CHECK-NEXT:    [[I_NEXT_10]] = add nuw i64 [[I_10]], 1
; CHECK-NEXT:    [[J_NEXT_10]] = add nuw i64 [[J_10]], 1
; CHECK-NEXT:    [[EC_10:%.*]] = icmp eq i64 [[I_NEXT_10]], [[W]]
; CHECK-NEXT:    br i1 [[EC_10]], label %[[LOOP_10_EXIT:.*]], label %[[LOOP_10]], !llvm.loop [[LOOP0]]
; CHECK:       [[LOOP_10_EXIT]]:
; CHECK-NEXT:    [[J_NEXT_10_LCSSA:%.*]] = phi i64 [ [[J_NEXT_10]], %[[LATCH_10]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[W]], i64 [[N]])
; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[W]]
; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[W]], -1
; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP1]], i64 [[TMP2]])
; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[W]], [[TMP3]]
; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 1
; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 [[TMP5]])
; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], -1
; CHECK-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], [[W]]
; CHECK-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], [[TMP3]]
; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP9]], i64 [[TMP2]])
; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[W]], [[TMP10]]
; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP11]], [[TMP3]]
; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], 2
; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 [[TMP13]])
; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP14]], -2
; CHECK-NEXT:    [[TMP16:%.*]] = sub i64 [[TMP15]], [[W]]
; CHECK-NEXT:    [[TMP17:%.*]] = sub i64 [[TMP16]], [[TMP10]]
; CHECK-NEXT:    [[TMP18:%.*]] = sub i64 [[TMP17]], [[TMP3]]
; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP18]], i64 [[TMP2]])
; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[W]], [[TMP19]]
; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[TMP20]], [[TMP10]]
; CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[TMP21]], [[TMP3]]
; CHECK-NEXT:    [[TMP23:%.*]] = add i64 [[TMP22]], 3
; CHECK-NEXT:    [[TMP24:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 [[TMP23]])
; CHECK-NEXT:    [[TMP25:%.*]] = add i64 [[TMP24]], -3
; CHECK-NEXT:    [[TMP26:%.*]] = sub i64 [[TMP25]], [[W]]
; CHECK-NEXT:    [[TMP27:%.*]] = sub i64 [[TMP26]], [[TMP19]]
; CHECK-NEXT:    [[TMP28:%.*]] = sub i64 [[TMP27]], [[TMP10]]
; CHECK-NEXT:    [[TMP29:%.*]] = sub i64 [[TMP28]], [[TMP3]]
; CHECK-NEXT:    [[TMP30:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP29]], i64 [[TMP2]])
; CHECK-NEXT:    [[TMP31:%.*]] = add i64 [[W]], [[TMP30]]
; CHECK-NEXT:    [[TMP32:%.*]] = add i64 [[TMP31]], [[TMP19]]
; CHECK-NEXT:    [[TMP33:%.*]] = add i64 [[TMP32]], [[TMP10]]
; CHECK-NEXT:    [[TMP34:%.*]] = add i64 [[TMP33]], [[TMP3]]
; CHECK-NEXT:    [[TMP35:%.*]] = add i64 [[TMP34]], 4
; CHECK-NEXT:    [[TMP36:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 [[TMP35]])
; CHECK-NEXT:    [[TMP37:%.*]] = add i64 [[TMP36]], -4
; CHECK-NEXT:    [[TMP38:%.*]] = sub i64 [[TMP37]], [[W]]
; CHECK-NEXT:    [[TMP39:%.*]] = sub i64 [[TMP38]], [[TMP30]]
; CHECK-NEXT:    [[TMP40:%.*]] = sub i64 [[TMP39]], [[TMP19]]
; CHECK-NEXT:    [[TMP41:%.*]] = sub i64 [[TMP40]], [[TMP10]]
; CHECK-NEXT:    [[TMP42:%.*]] = sub i64 [[TMP41]], [[TMP3]]
; CHECK-NEXT:    [[TMP43:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP42]], i64 [[TMP2]])
; CHECK-NEXT:    [[TMP44:%.*]] = add i64 [[W]], [[TMP43]]
; CHECK-NEXT:    [[TMP45:%.*]] = add i64 [[TMP44]], [[TMP30]]
; CHECK-NEXT:    [[TMP46:%.*]] = add i64 [[TMP45]], [[TMP19]]
; CHECK-NEXT:    [[TMP47:%.*]] = add i64 [[TMP46]], [[TMP10]]
; CHECK-NEXT:    [[TMP48:%.*]] = add i64 [[TMP47]], [[TMP3]]
; CHECK-NEXT:    [[TMP49:%.*]] = add i64 [[TMP48]], 5
; CHECK-NEXT:    [[TMP50:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 [[TMP49]])
; CHECK-NEXT:    [[TMP51:%.*]] = add i64 [[TMP50]], -5
; CHECK-NEXT:    [[TMP52:%.*]] = sub i64 [[TMP51]], [[W]]
; CHECK-NEXT:    [[TMP53:%.*]] = sub i64 [[TMP52]], [[TMP43]]
; CHECK-NEXT:    [[TMP54:%.*]] = sub i64 [[TMP53]], [[TMP30]]
; CHECK-NEXT:    [[TMP55:%.*]] = sub i64 [[TMP54]], [[TMP19]]
; CHECK-NEXT:    [[TMP56:%.*]] = sub i64 [[TMP55]], [[TMP10]]
; CHECK-NEXT:    [[TMP57:%.*]] = sub i64 [[TMP56]], [[TMP3]]
; CHECK-NEXT:    [[TMP58:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP57]], i64 [[TMP2]])
; CHECK-NEXT:    [[TMP59:%.*]] = add i64 [[W]], [[TMP58]]
; CHECK-NEXT:    [[TMP60:%.*]] = add i64 [[TMP59]], [[TMP43]]
; CHECK-NEXT:    [[TMP61:%.*]] = add i64 [[TMP60]], [[TMP30]]
; CHECK-NEXT:    [[TMP62:%.*]] = add i64 [[TMP61]], [[TMP19]]
; CHECK-NEXT:    [[TMP63:%.*]] = add i64 [[TMP62]], [[TMP10]]
; CHECK-NEXT:    [[TMP64:%.*]] = add i64 [[TMP63]], [[TMP3]]
; CHECK-NEXT:    [[TMP65:%.*]] = add i64 [[TMP64]], 6
; CHECK-NEXT:    [[TMP66:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 [[TMP65]])
; CHECK-NEXT:    [[TMP67:%.*]] = sub i64 0, [[TMP65]]
; CHECK-NEXT:    [[TMP68:%.*]] = sub i64 0, [[TMP67]]
; CHECK-NEXT:    [[TMP69:%.*]] = sub i64 [[TMP66]], [[TMP68]]
; CHECK-NEXT:    [[TMP70:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP69]], i64 [[TMP2]])
; CHECK-NEXT:    [[TMP71:%.*]] = add i64 [[TMP64]], 7
; CHECK-NEXT:    [[TMP72:%.*]] = add i64 [[TMP70]], [[TMP71]]
; CHECK-NEXT:    [[TMP73:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 [[TMP72]])
; CHECK-NEXT:    [[TMP74:%.*]] = sub i64 0, [[TMP72]]
; CHECK-NEXT:    [[TMP75:%.*]] = sub i64 0, [[TMP74]]
; CHECK-NEXT:    [[TMP76:%.*]] = sub i64 [[TMP73]], [[TMP75]]
; CHECK-NEXT:    [[TMP77:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP76]], i64 [[TMP2]])
; CHECK-NEXT:    [[TMP78:%.*]] = add i64 [[TMP64]], 8
; CHECK-NEXT:    [[TMP79:%.*]] = add i64 [[TMP70]], [[TMP78]]
; CHECK-NEXT:    [[TMP80:%.*]] = add i64 [[TMP77]], [[TMP79]]
; CHECK-NEXT:    [[TMP81:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 [[TMP80]])
; CHECK-NEXT:    [[TMP82:%.*]] = sub i64 0, [[TMP80]]
; CHECK-NEXT:    [[TMP83:%.*]] = sub i64 0, [[TMP82]]
; CHECK-NEXT:    [[TMP84:%.*]] = sub i64 [[TMP81]], [[TMP83]]
; CHECK-NEXT:    [[TMP85:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP84]], i64 [[TMP2]])
; CHECK-NEXT:    [[TMP86:%.*]] = add i64 [[TMP64]], 9
; CHECK-NEXT:    [[TMP87:%.*]] = add i64 [[TMP70]], [[TMP86]]
; CHECK-NEXT:    [[TMP88:%.*]] = add i64 [[TMP77]], [[TMP87]]
; CHECK-NEXT:    [[TMP89:%.*]] = add i64 [[TMP85]], [[TMP88]]
; CHECK-NEXT:    [[TMP90:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 [[TMP89]])
; CHECK-NEXT:    [[TMP91:%.*]] = sub i64 0, [[TMP89]]
; CHECK-NEXT:    [[TMP92:%.*]] = sub i64 0, [[TMP91]]
; CHECK-NEXT:    [[TMP93:%.*]] = sub i64 [[TMP90]], [[TMP92]]
; CHECK-NEXT:    [[TMP94:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP93]], i64 [[TMP2]])
; CHECK-NEXT:    [[TMP95:%.*]] = add i64 [[TMP64]], 10
; CHECK-NEXT:    [[TMP96:%.*]] = add i64 [[TMP70]], [[TMP95]]
; CHECK-NEXT:    [[TMP97:%.*]] = add i64 [[TMP77]], [[TMP96]]
; CHECK-NEXT:    [[TMP98:%.*]] = add i64 [[TMP85]], [[TMP97]]
; CHECK-NEXT:    [[TMP99:%.*]] = add i64 [[TMP94]], [[TMP98]]
; CHECK-NEXT:    [[TMP100:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 [[TMP99]])
; CHECK-NEXT:    [[TMP101:%.*]] = sub i64 0, [[TMP99]]
; CHECK-NEXT:    [[TMP102:%.*]] = sub i64 0, [[TMP101]]
; CHECK-NEXT:    [[TMP103:%.*]] = sub i64 [[TMP100]], [[TMP102]]
; CHECK-NEXT:    [[TMP104:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP103]], i64 [[TMP2]])
; CHECK-NEXT:    [[TMP105:%.*]] = add i64 [[TMP64]], 11
; CHECK-NEXT:    [[TMP106:%.*]] = add i64 [[TMP70]], [[TMP105]]
; CHECK-NEXT:    [[TMP107:%.*]] = add i64 [[TMP77]], [[TMP106]]
; CHECK-NEXT:    [[TMP108:%.*]] = add i64 [[TMP85]], [[TMP107]]
; CHECK-NEXT:    [[TMP109:%.*]] = add i64 [[TMP94]], [[TMP108]]
; CHECK-NEXT:    [[TMP110:%.*]] = add i64 [[TMP104]], [[TMP109]]
; CHECK-NEXT:    [[TMP111:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 [[TMP110]])
; CHECK-NEXT:    [[TMP112:%.*]] = sub i64 0, [[TMP110]]
; CHECK-NEXT:    [[TMP113:%.*]] = sub i64 0, [[TMP112]]
; CHECK-NEXT:    [[TMP114:%.*]] = sub i64 [[TMP111]], [[TMP113]]
; CHECK-NEXT:    [[TMP115:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP114]], i64 [[TMP2]])
; CHECK-NEXT:    [[TMP116:%.*]] = add i64 [[TMP115]], 1
; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP116]], 4
; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TMP117:%.*]] = and i64 [[TMP116]], 3
; CHECK-NEXT:    [[TMP118:%.*]] = icmp eq i64 [[TMP117]], 0
; CHECK-NEXT:    [[TMP119:%.*]] = select i1 [[TMP118]], i64 4, i64 [[TMP117]]
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP116]], [[TMP119]]
; CHECK-NEXT:    [[TMP120:%.*]] = add i64 [[J_NEXT_10_LCSSA]], [[N_VEC]]
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP121:%.*]] = add nuw i64 [[J_NEXT_10_LCSSA]], [[INDEX]]
; CHECK-NEXT:    [[TMP122:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 [[TMP121]]
; CHECK-NEXT:    store <4 x i8> zeroinitializer, ptr [[TMP122]], align 1
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP123:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP123]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[SCALAR_PH]]
; CHECK:       [[SCALAR_PH]]:
; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP120]], %[[MIDDLE_BLOCK]] ], [ [[J_NEXT_10_LCSSA]], %[[LOOP_10_EXIT]] ]
; CHECK-NEXT:    [[BC_RESUME_VAL12:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_10_EXIT]] ]
; CHECK-NEXT:    br label %[[LOOP_11:.*]]
; CHECK:       [[LOOP_11]]:
; CHECK-NEXT:    [[J_11:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[J_NEXT_11:%.*]], %[[LATCH_11:.*]] ]
; CHECK-NEXT:    [[I_11:%.*]] = phi i64 [ [[BC_RESUME_VAL12]], %[[SCALAR_PH]] ], [ [[I_NEXT_11:%.*]], %[[LATCH_11]] ]
; CHECK-NEXT:    [[C_11:%.*]] = icmp ult i64 [[J_11]], [[N]]
; CHECK-NEXT:    br i1 [[C_11]], label %[[LATCH_11]], label %[[EARLY_EXIT_LOOPEXIT:.*]]
; CHECK:       [[LATCH_11]]:
; CHECK-NEXT:    [[GEP_11:%.*]] = getelementptr inbounds nuw i8, ptr [[DST]], i64 [[J_11]]
; CHECK-NEXT:    store i8 0, ptr [[GEP_11]], align 1
; CHECK-NEXT:    [[I_NEXT_11]] = add nuw i64 [[I_11]], 1
; CHECK-NEXT:    [[J_NEXT_11]] = add nuw i64 [[J_11]], 1
; CHECK-NEXT:    [[EC_11:%.*]] = icmp eq i64 [[I_NEXT_11]], [[W]]
; CHECK-NEXT:    br i1 [[EC_11]], label %[[LOOP_11_EXIT:.*]], label %[[LOOP_11]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK:       [[LOOP_11_EXIT]]:
; CHECK-NEXT:    ret void
; CHECK:       [[EARLY_EXIT_LOOPEXIT]]:
; CHECK-NEXT:    br label %[[EARLY_EXIT:.*]]
; CHECK:       [[EARLY_EXIT_LOOPEXIT1]]:
; CHECK-NEXT:    br label %[[EARLY_EXIT]]
; CHECK:       [[EARLY_EXIT_LOOPEXIT2]]:
; CHECK-NEXT:    br label %[[EARLY_EXIT]]
; CHECK:       [[EARLY_EXIT_LOOPEXIT3]]:
; CHECK-NEXT:    br label %[[EARLY_EXIT]]
; CHECK:       [[EARLY_EXIT_LOOPEXIT4]]:
; CHECK-NEXT:    br label %[[EARLY_EXIT]]
; CHECK:       [[EARLY_EXIT_LOOPEXIT5]]:
; CHECK-NEXT:    br label %[[EARLY_EXIT]]
; CHECK:       [[EARLY_EXIT_LOOPEXIT6]]:
; CHECK-NEXT:    br label %[[EARLY_EXIT]]
; CHECK:       [[EARLY_EXIT_LOOPEXIT7]]:
; CHECK-NEXT:    br label %[[EARLY_EXIT]]
; CHECK:       [[EARLY_EXIT_LOOPEXIT8]]:
; CHECK-NEXT:    br label %[[EARLY_EXIT]]
; CHECK:       [[EARLY_EXIT_LOOPEXIT9]]:
; CHECK-NEXT:    br label %[[EARLY_EXIT]]
; CHECK:       [[EARLY_EXIT_LOOPEXIT10]]:
; CHECK-NEXT:    br label %[[EARLY_EXIT]]
; CHECK:       [[EARLY_EXIT_LOOPEXIT11]]:
; CHECK-NEXT:    br label %[[EARLY_EXIT]]
; CHECK:       [[EARLY_EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %loop.0

loop.0:
  %j.0 = phi i64 [ %w, %entry ], [ %j.next.0, %latch.0 ]
  %i.0 = phi i64 [ 0, %entry ], [ %i.next.0, %latch.0 ]
  %c.0 = icmp ult i64 %j.0, %n
  br i1 %c.0, label %latch.0, label %early.exit

latch.0:
  %i.next.0 = add nuw i64 %i.0, 1
  %j.next.0 = add nuw i64 %j.0, 1
  %ec.0 = icmp eq i64 %i.next.0, %w
  br i1 %ec.0, label %loop.0.exit, label %loop.0, !llvm.loop !0

loop.0.exit:
  %j.next.0.lcssa = phi i64 [ %j.next.0, %latch.0 ]
  br label %loop.1

loop.1:
  %j.1 = phi i64 [ %j.next.0.lcssa, %loop.0.exit ], [ %j.next.1, %latch.1 ]
  %i.1 = phi i64 [ 0, %loop.0.exit ], [ %i.next.1, %latch.1 ]
  %c.1 = icmp ult i64 %j.1, %n
  br i1 %c.1, label %latch.1, label %early.exit

latch.1:
  %i.next.1 = add nuw i64 %i.1, 1
  %j.next.1 = add nuw i64 %j.1, 1
  %ec.1 = icmp eq i64 %i.next.1, %w
  br i1 %ec.1, label %loop.1.exit, label %loop.1, !llvm.loop !0

loop.1.exit:
  %j.next.1.lcssa = phi i64 [ %j.next.1, %latch.1 ]
  br label %loop.2

loop.2:
  %j.2 = phi i64 [ %j.next.1.lcssa, %loop.1.exit ], [ %j.next.2, %latch.2 ]
  %i.2 = phi i64 [ 0, %loop.1.exit ], [ %i.next.2, %latch.2 ]
  %c.2 = icmp ult i64 %j.2, %n
  br i1 %c.2, label %latch.2, label %early.exit

latch.2:
  %i.next.2 = add nuw i64 %i.2, 1
  %j.next.2 = add nuw i64 %j.2, 1
  %ec.2 = icmp eq i64 %i.next.2, %w
  br i1 %ec.2, label %loop.2.exit, label %loop.2, !llvm.loop !0

loop.2.exit:
  %j.next.2.lcssa = phi i64 [ %j.next.2, %latch.2 ]
  br label %loop.3

loop.3:
  %j.3 = phi i64 [ %j.next.2.lcssa, %loop.2.exit ], [ %j.next.3, %latch.3 ]
  %i.3 = phi i64 [ 0, %loop.2.exit ], [ %i.next.3, %latch.3 ]
  %c.3 = icmp ult i64 %j.3, %n
  br i1 %c.3, label %latch.3, label %early.exit

latch.3:
  %i.next.3 = add nuw i64 %i.3, 1
  %j.next.3 = add nuw i64 %j.3, 1
  %ec.3 = icmp eq i64 %i.next.3, %w
  br i1 %ec.3, label %loop.3.exit, label %loop.3, !llvm.loop !0

loop.3.exit:
  %j.next.3.lcssa = phi i64 [ %j.next.3, %latch.3 ]
  br label %loop.4

loop.4:
  %j.4 = phi i64 [ %j.next.3.lcssa, %loop.3.exit ], [ %j.next.4, %latch.4 ]
  %i.4 = phi i64 [ 0, %loop.3.exit ], [ %i.next.4, %latch.4 ]
  %c.4 = icmp ult i64 %j.4, %n
  br i1 %c.4, label %latch.4, label %early.exit

latch.4:
  %i.next.4 = add nuw i64 %i.4, 1
  %j.next.4 = add nuw i64 %j.4, 1
  %ec.4 = icmp eq i64 %i.next.4, %w
  br i1 %ec.4, label %loop.4.exit, label %loop.4, !llvm.loop !0

loop.4.exit:
  %j.next.4.lcssa = phi i64 [ %j.next.4, %latch.4 ]
  br label %loop.5

loop.5:
  %j.5 = phi i64 [ %j.next.4.lcssa, %loop.4.exit ], [ %j.next.5, %latch.5 ]
  %i.5 = phi i64 [ 0, %loop.4.exit ], [ %i.next.5, %latch.5 ]
  %c.5 = icmp ult i64 %j.5, %n
  br i1 %c.5, label %latch.5, label %early.exit

latch.5:
  %i.next.5 = add nuw i64 %i.5, 1
  %j.next.5 = add nuw i64 %j.5, 1
  %ec.5 = icmp eq i64 %i.next.5, %w
  br i1 %ec.5, label %loop.5.exit, label %loop.5, !llvm.loop !0

loop.5.exit:
  %j.next.5.lcssa = phi i64 [ %j.next.5, %latch.5 ]
  br label %loop.6

loop.6:
  %j.6 = phi i64 [ %j.next.5.lcssa, %loop.5.exit ], [ %j.next.6, %latch.6 ]
  %i.6 = phi i64 [ 0, %loop.5.exit ], [ %i.next.6, %latch.6 ]
  %c.6 = icmp ult i64 %j.6, %n
  br i1 %c.6, label %latch.6, label %early.exit

latch.6:
  %i.next.6 = add nuw i64 %i.6, 1
  %j.next.6 = add nuw i64 %j.6, 1
  %ec.6 = icmp eq i64 %i.next.6, %w
  br i1 %ec.6, label %loop.6.exit, label %loop.6, !llvm.loop !0

loop.6.exit:
  %j.next.6.lcssa = phi i64 [ %j.next.6, %latch.6 ]
  br label %loop.7

loop.7:
  %j.7 = phi i64 [ %j.next.6.lcssa, %loop.6.exit ], [ %j.next.7, %latch.7 ]
  %i.7 = phi i64 [ 0, %loop.6.exit ], [ %i.next.7, %latch.7 ]
  %c.7 = icmp ult i64 %j.7, %n
  br i1 %c.7, label %latch.7, label %early.exit

latch.7:
  %i.next.7 = add nuw i64 %i.7, 1
  %j.next.7 = add nuw i64 %j.7, 1
  %ec.7 = icmp eq i64 %i.next.7, %w
  br i1 %ec.7, label %loop.7.exit, label %loop.7, !llvm.loop !0

loop.7.exit:
  %j.next.7.lcssa = phi i64 [ %j.next.7, %latch.7 ]
  br label %loop.8

loop.8:
  %j.8 = phi i64 [ %j.next.7.lcssa, %loop.7.exit ], [ %j.next.8, %latch.8 ]
  %i.8 = phi i64 [ 0, %loop.7.exit ], [ %i.next.8, %latch.8 ]
  %c.8 = icmp ult i64 %j.8, %n
  br i1 %c.8, label %latch.8, label %early.exit

latch.8:
  %i.next.8 = add nuw i64 %i.8, 1
  %j.next.8 = add nuw i64 %j.8, 1
  %ec.8 = icmp eq i64 %i.next.8, %w
  br i1 %ec.8, label %loop.8.exit, label %loop.8, !llvm.loop !0

loop.8.exit:
  %j.next.8.lcssa = phi i64 [ %j.next.8, %latch.8 ]
  br label %loop.9

loop.9:
  %j.9 = phi i64 [ %j.next.8.lcssa, %loop.8.exit ], [ %j.next.9, %latch.9 ]
  %i.9 = phi i64 [ 0, %loop.8.exit ], [ %i.next.9, %latch.9 ]
  %c.9 = icmp ult i64 %j.9, %n
  br i1 %c.9, label %latch.9, label %early.exit

latch.9:
  %i.next.9 = add nuw i64 %i.9, 1
  %j.next.9 = add nuw i64 %j.9, 1
  %ec.9 = icmp eq i64 %i.next.9, %w
  br i1 %ec.9, label %loop.9.exit, label %loop.9, !llvm.loop !0

loop.9.exit:
  %j.next.9.lcssa = phi i64 [ %j.next.9, %latch.9 ]
  br label %loop.10

loop.10:
  %j.10 = phi i64 [ %j.next.9.lcssa, %loop.9.exit ], [ %j.next.10, %latch.10 ]
  %i.10 = phi i64 [ 0, %loop.9.exit ], [ %i.next.10, %latch.10 ]
  %c.10 = icmp ult i64 %j.10, %n
  br i1 %c.10, label %latch.10, label %early.exit

latch.10:
  %i.next.10 = add nuw i64 %i.10, 1
  %j.next.10 = add nuw i64 %j.10, 1
  %ec.10 = icmp eq i64 %i.next.10, %w
  br i1 %ec.10, label %loop.10.exit, label %loop.10, !llvm.loop !0

loop.10.exit:
  %j.next.10.lcssa = phi i64 [ %j.next.10, %latch.10 ]
  br label %loop.11

loop.11:
  %j.11 = phi i64 [ %j.next.10.lcssa, %loop.10.exit ], [ %j.next.11, %latch.11 ]
  %i.11 = phi i64 [ 0, %loop.10.exit ], [ %i.next.11, %latch.11 ]
  %c.11 = icmp ult i64 %j.11, %n
  br i1 %c.11, label %latch.11, label %early.exit

latch.11:
  %gep.11 = getelementptr inbounds nuw i8, ptr %dst, i64 %j.11
  store i8 0, ptr %gep.11, align 1
  %i.next.11 = add nuw i64 %i.11, 1
  %j.next.11 = add nuw i64 %j.11, 1
  %ec.11 = icmp eq i64 %i.next.11, %w
  br i1 %ec.11, label %loop.11.exit, label %loop.11

loop.11.exit:
  ret void

early.exit:
  ret void
}

!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.vectorize.width", i32 1}