llvm/test/Transforms/LoopVectorize/X86/small-size.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -loop-vectorize-with-block-frequency -S | FileCheck %s

target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
target triple = "x86_64-apple-macosx10.8.0"

@b = common global [2048 x i32] zeroinitializer, align 16
@c = common global [2048 x i32] zeroinitializer, align 16
@a = common global [2048 x i32] zeroinitializer, align 16
@G = common global [32 x [1024 x i32]] zeroinitializer, align 16
@ub = common global [1024 x i32] zeroinitializer, align 16
@uc = common global [1024 x i32] zeroinitializer, align 16
@d = common global [2048 x i32] zeroinitializer, align 16
@fa = common global [1024 x float] zeroinitializer, align 16
@fb = common global [1024 x float] zeroinitializer, align 16
@ic = common global [1024 x i32] zeroinitializer, align 16
@da = common global [1024 x float] zeroinitializer, align 16
@db = common global [1024 x float] zeroinitializer, align 16
@dc = common global [1024 x float] zeroinitializer, align 16
@dd = common global [1024 x float] zeroinitializer, align 16
@dj = common global [1024 x i32] zeroinitializer, align 16

; We can optimize this test without a tail.
define void @example1() optsize {
; CHECK-LABEL: @example1(
; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]
; CHECK:       vector.ph:
; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
; CHECK:       vector.body:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 [[INDEX]]
; CHECK-NEXT:    store <4 x i32> [[TMP3]], ptr [[TMP4]], align 4
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       middle.block:
; CHECK-NEXT:    br label [[TMP6:%.*]]
; CHECK:       6:
; CHECK-NEXT:    ret void
;
  br label %1

; <label>:
  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
  %2 = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 %indvars.iv
  %3 = load i32, ptr %2, align 4
  %4 = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 %indvars.iv
  %5 = load i32, ptr %4, align 4
  %6 = add nsw i32 %5, %3
  %7 = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 %indvars.iv
  store i32 %6, ptr %7, align 4
  %indvars.iv.next = add i64 %indvars.iv, 1
  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
  %exitcond = icmp eq i32 %lftr.wideiv, 256
  br i1 %exitcond, label %8, label %1

; <label>:
  ret void
}

; Can vectorize in 'optsize' mode by masking the needed tail.
define void @example2(i32 %n, i32 %x) optsize {
; CHECK-LABEL: @example2(
; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[N:%.*]], 0
; CHECK-NEXT:    br i1 [[TMP1]], label [[DOTLR_PH5_PREHEADER:%.*]], label [[DOTPREHEADER:%.*]]
; CHECK:       .lr.ph5.preheader:
; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[N]] to i64
; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]
; CHECK:       vector.ph:
; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP2]], 3
; CHECK-NEXT:    [[TMP3:%.*]] = and i64 [[N_RND_UP]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP3]]
; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP2]], 1
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
; CHECK:       vector.body:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
; CHECK-NEXT:    [[TMP4:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0
; CHECK-NEXT:    br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK:       pred.store.if:
; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[INDEX]]
; CHECK-NEXT:    store i32 [[X:%.*]], ptr [[TMP6]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
; CHECK:       pred.store.continue:
; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1
; CHECK-NEXT:    br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
; CHECK:       pred.store.if1:
; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[TMP8]]
; CHECK-NEXT:    store i32 [[X]], ptr [[TMP9]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE2]]
; CHECK:       pred.store.continue2:
; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2
; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
; CHECK:       pred.store.if3:
; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[TMP11]]
; CHECK-NEXT:    store i32 [[X]], ptr [[TMP12]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE4]]
; CHECK:       pred.store.continue4:
; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3
; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
; CHECK:       pred.store.if5:
; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[TMP14]]
; CHECK-NEXT:    store i32 [[X]], ptr [[TMP15]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
; CHECK:       pred.store.continue6:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK:       middle.block:
; CHECK-NEXT:    br label [[DOT_PREHEADER_CRIT_EDGE:%.*]]
; CHECK:       ..preheader_crit_edge:
; CHECK-NEXT:    [[PHITMP:%.*]] = sext i32 [[N]] to i64
; CHECK-NEXT:    br label [[DOTPREHEADER]]
; CHECK:       .preheader:
; CHECK-NEXT:    [[I_0_LCSSA:%.*]] = phi i64 [ [[PHITMP]], [[DOT_PREHEADER_CRIT_EDGE]] ], [ 0, [[TMP0:%.*]] ]
; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i32 [[N]], 0
; CHECK-NEXT:    br i1 [[TMP17]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH_PREHEADER:%.*]]
; CHECK:       .lr.ph.preheader:
; CHECK-NEXT:    [[TMP18:%.*]] = zext i32 [[N]] to i64
; CHECK-NEXT:    br label [[VECTOR_PH7:%.*]]
; CHECK:       vector.ph7:
; CHECK-NEXT:    [[N_RND_UP8:%.*]] = add i64 [[TMP18]], 3
; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[N_RND_UP8]], 3
; CHECK-NEXT:    [[N_VEC9:%.*]] = sub i64 [[N_RND_UP8]], [[TMP19]]
; CHECK-NEXT:    [[TRIP_COUNT_MINUS_110:%.*]] = sub i64 [[TMP18]], 1
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_110]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT12:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT11]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label [[VECTOR_BODY13:%.*]]
; CHECK:       vector.body13:
; CHECK-NEXT:    [[INDEX14:%.*]] = phi i64 [ 0, [[VECTOR_PH7]] ], [ [[INDEX_NEXT24:%.*]], [[PRED_STORE_CONTINUE23:%.*]] ]
; CHECK-NEXT:    [[VEC_IND15:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH7]] ], [ [[VEC_IND_NEXT25:%.*]], [[PRED_STORE_CONTINUE23]] ]
; CHECK-NEXT:    [[TMP20:%.*]] = icmp ule <4 x i64> [[VEC_IND15]], [[BROADCAST_SPLAT12]]
; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[I_0_LCSSA]], [[INDEX14]]
; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[TMP20]], i64 0
; CHECK-NEXT:    br i1 [[TMP22]], label [[PRED_STORE_IF16:%.*]], label [[PRED_STORE_CONTINUE17:%.*]]
; CHECK:       pred.store.if16:
; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[TMP21]]
; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4
; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 [[TMP21]]
; CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4
; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 [[TMP21]]
; CHECK-NEXT:    [[TMP28:%.*]] = and i32 [[TMP26]], [[TMP24]]
; CHECK-NEXT:    store i32 [[TMP28]], ptr [[TMP27]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE17]]
; CHECK:       pred.store.continue17:
; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x i1> [[TMP20]], i64 1
; CHECK-NEXT:    br i1 [[TMP29]], label [[PRED_STORE_IF18:%.*]], label [[PRED_STORE_CONTINUE19:%.*]]
; CHECK:       pred.store.if18:
; CHECK-NEXT:    [[TMP30:%.*]] = add i64 [[TMP21]], 1
; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[TMP30]]
; CHECK-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP31]], align 4
; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 [[TMP30]]
; CHECK-NEXT:    [[TMP34:%.*]] = load i32, ptr [[TMP33]], align 4
; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 [[TMP30]]
; CHECK-NEXT:    [[TMP36:%.*]] = and i32 [[TMP34]], [[TMP32]]
; CHECK-NEXT:    store i32 [[TMP36]], ptr [[TMP35]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE19]]
; CHECK:       pred.store.continue19:
; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <4 x i1> [[TMP20]], i64 2
; CHECK-NEXT:    br i1 [[TMP37]], label [[PRED_STORE_IF20:%.*]], label [[PRED_STORE_CONTINUE21:%.*]]
; CHECK:       pred.store.if20:
; CHECK-NEXT:    [[TMP38:%.*]] = add i64 [[TMP21]], 2
; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[TMP38]]
; CHECK-NEXT:    [[TMP40:%.*]] = load i32, ptr [[TMP39]], align 4
; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 [[TMP38]]
; CHECK-NEXT:    [[TMP42:%.*]] = load i32, ptr [[TMP41]], align 4
; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 [[TMP38]]
; CHECK-NEXT:    [[TMP44:%.*]] = and i32 [[TMP42]], [[TMP40]]
; CHECK-NEXT:    store i32 [[TMP44]], ptr [[TMP43]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE21]]
; CHECK:       pred.store.continue21:
; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <4 x i1> [[TMP20]], i64 3
; CHECK-NEXT:    br i1 [[TMP45]], label [[PRED_STORE_IF22:%.*]], label [[PRED_STORE_CONTINUE23]]
; CHECK:       pred.store.if22:
; CHECK-NEXT:    [[TMP46:%.*]] = add i64 [[TMP21]], 3
; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 [[TMP46]]
; CHECK-NEXT:    [[TMP48:%.*]] = load i32, ptr [[TMP47]], align 4
; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 [[TMP46]]
; CHECK-NEXT:    [[TMP50:%.*]] = load i32, ptr [[TMP49]], align 4
; CHECK-NEXT:    [[TMP51:%.*]] = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 [[TMP46]]
; CHECK-NEXT:    [[TMP52:%.*]] = and i32 [[TMP50]], [[TMP48]]
; CHECK-NEXT:    store i32 [[TMP52]], ptr [[TMP51]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE23]]
; CHECK:       pred.store.continue23:
; CHECK-NEXT:    [[INDEX_NEXT24]] = add nuw i64 [[INDEX14]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT25]] = add nuw <4 x i64> [[VEC_IND15]], splat (i64 4)
; CHECK-NEXT:    [[TMP53:%.*]] = icmp eq i64 [[INDEX_NEXT24]], [[N_VEC9]]
; CHECK-NEXT:    br i1 [[TMP53]], label [[MIDDLE_BLOCK26:%.*]], label [[VECTOR_BODY13]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK:       middle.block26:
; CHECK-NEXT:    br label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]]
; CHECK:       ._crit_edge.loopexit:
; CHECK-NEXT:    br label [[DOT_CRIT_EDGE]]
; CHECK:       ._crit_edge:
; CHECK-NEXT:    ret void
;
  %1 = icmp sgt i32 %n, 0
  br i1 %1, label %.lr.ph5, label %.preheader

..preheader_crit_edge:
  %phitmp = sext i32 %n to i64
  br label %.preheader

.preheader:
  %i.0.lcssa = phi i64 [ %phitmp, %..preheader_crit_edge ], [ 0, %0 ]
  %2 = icmp eq i32 %n, 0
  br i1 %2, label %._crit_edge, label %.lr.ph

.lr.ph5:
  %indvars.iv6 = phi i64 [ %indvars.iv.next7, %.lr.ph5 ], [ 0, %0 ]
  %3 = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 %indvars.iv6
  store i32 %x, ptr %3, align 4
  %indvars.iv.next7 = add i64 %indvars.iv6, 1
  %lftr.wideiv = trunc i64 %indvars.iv.next7 to i32
  %exitcond = icmp eq i32 %lftr.wideiv, %n
  br i1 %exitcond, label %..preheader_crit_edge, label %.lr.ph5

.lr.ph:
  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ %i.0.lcssa, %.preheader ]
  %.02 = phi i32 [ %4, %.lr.ph ], [ %n, %.preheader ]
  %4 = add nsw i32 %.02, -1
  %5 = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 %indvars.iv
  %6 = load i32, ptr %5, align 4
  %7 = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 %indvars.iv
  %8 = load i32, ptr %7, align 4
  %9 = and i32 %8, %6
  %10 = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 %indvars.iv
  store i32 %9, ptr %10, align 4
  %indvars.iv.next = add i64 %indvars.iv, 1
  %11 = icmp eq i32 %4, 0
  br i1 %11, label %._crit_edge, label %.lr.ph

._crit_edge:
  ret void
}

; Loop has no primary induction as its integer IV has step -1 starting at
; unknown N, but can still be vectorized.
; Note: Most scalar pointer induction GEPs could be sunk into the conditional
;       blocks.
define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q) optsize {
; CHECK-LABEL: @example3(
; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[N:%.*]], 0
; CHECK-NEXT:    br i1 [[TMP1]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH_PREHEADER:%.*]]
; CHECK:       .lr.ph.preheader:
; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[N]] to i64
; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]
; CHECK:       vector.ph:
; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP2]], 3
; CHECK-NEXT:    [[TMP3:%.*]] = and i64 [[N_RND_UP]], 3
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[TMP3]]
; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP2]], 1
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
; CHECK:       vector.body:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE13:%.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE13]] ]
; CHECK-NEXT:    [[TMP4:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[INDEX]], 2
; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP5]], 4
; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP5]], 8
; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP5]], 12
; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[TMP5]]
; CHECK-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP6]]
; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP7]]
; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP8]]
; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[Q:%.*]], i64 [[TMP5]]
; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP6]]
; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP7]]
; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP8]]
; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0
; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK:       pred.store.if:
; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[NEXT_GEP4]], align 16
; CHECK-NEXT:    store i32 [[TMP10]], ptr [[NEXT_GEP]], align 16
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
; CHECK:       pred.store.continue:
; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1
; CHECK-NEXT:    br i1 [[TMP11]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
; CHECK:       pred.store.if8:
; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[NEXT_GEP5]], align 16
; CHECK-NEXT:    store i32 [[TMP12]], ptr [[NEXT_GEP1]], align 16
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE9]]
; CHECK:       pred.store.continue9:
; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2
; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
; CHECK:       pred.store.if10:
; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[NEXT_GEP6]], align 16
; CHECK-NEXT:    store i32 [[TMP14]], ptr [[NEXT_GEP2]], align 16
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE11]]
; CHECK:       pred.store.continue11:
; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3
; CHECK-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13]]
; CHECK:       pred.store.if12:
; CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[NEXT_GEP7]], align 16
; CHECK-NEXT:    store i32 [[TMP16]], ptr [[NEXT_GEP3]], align 16
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE13]]
; CHECK:       pred.store.continue13:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK:       middle.block:
; CHECK-NEXT:    br label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]]
; CHECK:       ._crit_edge.loopexit:
; CHECK-NEXT:    br label [[DOT_CRIT_EDGE]]
; CHECK:       ._crit_edge:
; CHECK-NEXT:    ret void
;
  %1 = icmp eq i32 %n, 0
  br i1 %1, label %._crit_edge, label %.lr.ph

.lr.ph:
  %.05 = phi i32 [ %2, %.lr.ph ], [ %n, %0 ]
  %.014 = phi ptr [ %5, %.lr.ph ], [ %p, %0 ]
  %.023 = phi ptr [ %3, %.lr.ph ], [ %q, %0 ]
  %2 = add nsw i32 %.05, -1
  %3 = getelementptr inbounds i32, ptr %.023, i64 1
  %4 = load i32, ptr %.023, align 16
  %5 = getelementptr inbounds i32, ptr %.014, i64 1
  store i32 %4, ptr %.014, align 16
  %6 = icmp eq i32 %2, 0
  br i1 %6, label %._crit_edge, label %.lr.ph

._crit_edge:
  ret void
}

; We can't vectorize this one because we need a runtime ptr check.
define void @example23(ptr nocapture %src, ptr nocapture %dst) optsize {
; CHECK-LABEL: @example23(
; CHECK-NEXT:    br label [[TMP1:%.*]]
; CHECK:       1:
; CHECK-NEXT:    [[DOT04:%.*]] = phi ptr [ [[SRC:%.*]], [[TMP0:%.*]] ], [ [[TMP2:%.*]], [[TMP1]] ]
; CHECK-NEXT:    [[DOT013:%.*]] = phi ptr [ [[DST:%.*]], [[TMP0]] ], [ [[TMP6:%.*]], [[TMP1]] ]
; CHECK-NEXT:    [[I_02:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[TMP7:%.*]], [[TMP1]] ]
; CHECK-NEXT:    [[TMP2]] = getelementptr inbounds i16, ptr [[DOT04]], i64 1
; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr [[DOT04]], align 2
; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP3]] to i32
; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 7
; CHECK-NEXT:    [[TMP6]] = getelementptr inbounds i32, ptr [[DOT013]], i64 1
; CHECK-NEXT:    store i32 [[TMP5]], ptr [[DOT013]], align 4
; CHECK-NEXT:    [[TMP7]] = add nsw i32 [[I_02]], 1
; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP7]], 256
; CHECK-NEXT:    br i1 [[EXITCOND]], label [[TMP8:%.*]], label [[TMP1]]
; CHECK:       8:
; CHECK-NEXT:    ret void
;
  br label %1

; <label>:
  %.04 = phi ptr [ %src, %0 ], [ %2, %1 ]
  %.013 = phi ptr [ %dst, %0 ], [ %6, %1 ]
  %i.02 = phi i32 [ 0, %0 ], [ %7, %1 ]
  %2 = getelementptr inbounds i16, ptr %.04, i64 1
  %3 = load i16, ptr %.04, align 2
  %4 = zext i16 %3 to i32
  %5 = shl nuw nsw i32 %4, 7
  %6 = getelementptr inbounds i32, ptr %.013, i64 1
  store i32 %5, ptr %.013, align 4
  %7 = add nsw i32 %i.02, 1
  %exitcond = icmp eq i32 %7, 256
  br i1 %exitcond, label %8, label %1

; <label>:
  ret void
}


; We CAN vectorize this example because the pointers are marked as noalias.
define void @example23b(ptr noalias nocapture %src, ptr noalias nocapture %dst) optsize {
; CHECK-LABEL: @example23b(
; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]
; CHECK:       vector.ph:
; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
; CHECK:       vector.body:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[TMP1]]
; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[INDEX]], 2
; CHECK-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[TMP2]]
; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[NEXT_GEP]], align 2
; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[WIDE_LOAD]] to <4 x i32>
; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw <4 x i32> [[TMP3]], splat (i32 7)
; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[NEXT_GEP1]], align 4
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK:       middle.block:
; CHECK-NEXT:    br label [[TMP6:%.*]]
; CHECK:       6:
; CHECK-NEXT:    ret void
;
  br label %1

; <label>:
  %.04 = phi ptr [ %src, %0 ], [ %2, %1 ]
  %.013 = phi ptr [ %dst, %0 ], [ %6, %1 ]
  %i.02 = phi i32 [ 0, %0 ], [ %7, %1 ]
  %2 = getelementptr inbounds i16, ptr %.04, i64 1
  %3 = load i16, ptr %.04, align 2
  %4 = zext i16 %3 to i32
  %5 = shl nuw nsw i32 %4, 7
  %6 = getelementptr inbounds i32, ptr %.013, i64 1
  store i32 %5, ptr %.013, align 4
  %7 = add nsw i32 %i.02, 1
  %exitcond = icmp eq i32 %7, 256
  br i1 %exitcond, label %8, label %1

; <label>:
  ret void
}

; We CAN vectorize this example by folding the tail it entails.
; Note: Most scalar pointer induction GEPs could be sunk into the conditional
;       blocks.
define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst) optsize {
; CHECK-LABEL: @example23c(
; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]
; CHECK:       vector.ph:
; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
; CHECK:       vector.body:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE13:%.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE13]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i16> [[VEC_IND]], splat (i16 256)
; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], 2
; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP2]], 4
; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP2]], 6
; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[TMP2]]
; CHECK-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP4]]
; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP5]]
; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 2
; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], 4
; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP6]], 8
; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP6]], 12
; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[TMP6]]
; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP9]]
; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK:       pred.store.if:
; CHECK-NEXT:    [[TMP11:%.*]] = load i16, ptr [[NEXT_GEP]], align 2
; CHECK-NEXT:    [[TMP12:%.*]] = zext i16 [[TMP11]] to i32
; CHECK-NEXT:    [[TMP13:%.*]] = shl nuw nsw i32 [[TMP12]], 7
; CHECK-NEXT:    store i32 [[TMP13]], ptr [[NEXT_GEP4]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
; CHECK:       pred.store.continue:
; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
; CHECK:       pred.store.if8:
; CHECK-NEXT:    [[TMP15:%.*]] = load i16, ptr [[NEXT_GEP1]], align 2
; CHECK-NEXT:    [[TMP16:%.*]] = zext i16 [[TMP15]] to i32
; CHECK-NEXT:    [[TMP17:%.*]] = shl nuw nsw i32 [[TMP16]], 7
; CHECK-NEXT:    store i32 [[TMP17]], ptr [[NEXT_GEP5]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE9]]
; CHECK:       pred.store.continue9:
; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
; CHECK-NEXT:    br i1 [[TMP18]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
; CHECK:       pred.store.if10:
; CHECK-NEXT:    [[TMP19:%.*]] = load i16, ptr [[NEXT_GEP2]], align 2
; CHECK-NEXT:    [[TMP20:%.*]] = zext i16 [[TMP19]] to i32
; CHECK-NEXT:    [[TMP21:%.*]] = shl nuw nsw i32 [[TMP20]], 7
; CHECK-NEXT:    store i32 [[TMP21]], ptr [[NEXT_GEP6]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE11]]
; CHECK:       pred.store.continue11:
; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
; CHECK-NEXT:    br i1 [[TMP22]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13]]
; CHECK:       pred.store.if12:
; CHECK-NEXT:    [[TMP23:%.*]] = load i16, ptr [[NEXT_GEP3]], align 2
; CHECK-NEXT:    [[TMP24:%.*]] = zext i16 [[TMP23]] to i32
; CHECK-NEXT:    [[TMP25:%.*]] = shl nuw nsw i32 [[TMP24]], 7
; CHECK-NEXT:    store i32 [[TMP25]], ptr [[NEXT_GEP7]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE13]]
; CHECK:       pred.store.continue13:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i16> [[VEC_IND]], splat (i16 4)
; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
; CHECK-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK:       middle.block:
; CHECK-NEXT:    br label [[TMP27:%.*]]
; CHECK:       27:
; CHECK-NEXT:    ret void
;
  br label %1

; <label>:
  %.04 = phi ptr [ %src, %0 ], [ %2, %1 ]
  %.013 = phi ptr [ %dst, %0 ], [ %6, %1 ]
  %i.02 = phi i64 [ 0, %0 ], [ %7, %1 ]
  %2 = getelementptr inbounds i16, ptr %.04, i64 1
  %3 = load i16, ptr %.04, align 2
  %4 = zext i16 %3 to i32
  %5 = shl nuw nsw i32 %4, 7
  %6 = getelementptr inbounds i32, ptr %.013, i64 1
  store i32 %5, ptr %.013, align 4
  %7 = add nsw i64 %i.02, 1
  %exitcond = icmp eq i64 %7, 257
  br i1 %exitcond, label %8, label %1

; <label>:
  ret void
}

; We CAN'T vectorize this example because it would entail a tail and an
; induction is used outside the loop.
define i64 @example23d(ptr noalias nocapture %src, ptr noalias nocapture %dst) optsize {
; CHECK-LABEL: @example23d(
; CHECK-NEXT:    br label [[VECTOR_PH:%.*]]
; CHECK:       vector.ph:
; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
; CHECK:       vector.body:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE13:%.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE13]] ]
; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i16> [[VEC_IND]], splat (i16 256)
; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], 2
; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP2]], 4
; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP2]], 6
; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[TMP2]]
; CHECK-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP4]]
; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP5]]
; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 2
; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], 4
; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP6]], 8
; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP6]], 12
; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[TMP6]]
; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP9]]
; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK:       pred.store.if:
; CHECK-NEXT:    [[TMP11:%.*]] = load i16, ptr [[NEXT_GEP]], align 2
; CHECK-NEXT:    [[TMP12:%.*]] = zext i16 [[TMP11]] to i32
; CHECK-NEXT:    [[TMP13:%.*]] = shl nuw nsw i32 [[TMP12]], 7
; CHECK-NEXT:    store i32 [[TMP13]], ptr [[NEXT_GEP4]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
; CHECK:       pred.store.continue:
; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
; CHECK:       pred.store.if8:
; CHECK-NEXT:    [[TMP15:%.*]] = load i16, ptr [[NEXT_GEP1]], align 2
; CHECK-NEXT:    [[TMP16:%.*]] = zext i16 [[TMP15]] to i32
; CHECK-NEXT:    [[TMP17:%.*]] = shl nuw nsw i32 [[TMP16]], 7
; CHECK-NEXT:    store i32 [[TMP17]], ptr [[NEXT_GEP5]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE9]]
; CHECK:       pred.store.continue9:
; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
; CHECK-NEXT:    br i1 [[TMP18]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
; CHECK:       pred.store.if10:
; CHECK-NEXT:    [[TMP19:%.*]] = load i16, ptr [[NEXT_GEP2]], align 2
; CHECK-NEXT:    [[TMP20:%.*]] = zext i16 [[TMP19]] to i32
; CHECK-NEXT:    [[TMP21:%.*]] = shl nuw nsw i32 [[TMP20]], 7
; CHECK-NEXT:    store i32 [[TMP21]], ptr [[NEXT_GEP6]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE11]]
; CHECK:       pred.store.continue11:
; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
; CHECK-NEXT:    br i1 [[TMP22]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13]]
; CHECK:       pred.store.if12:
; CHECK-NEXT:    [[TMP23:%.*]] = load i16, ptr [[NEXT_GEP3]], align 2
; CHECK-NEXT:    [[TMP24:%.*]] = zext i16 [[TMP23]] to i32
; CHECK-NEXT:    [[TMP25:%.*]] = shl nuw nsw i32 [[TMP24]], 7
; CHECK-NEXT:    store i32 [[TMP25]], ptr [[NEXT_GEP7]], align 4
; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE13]]
; CHECK:       pred.store.continue13:
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i16> [[VEC_IND]], splat (i16 4)
; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
; CHECK-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK:       middle.block:
; CHECK-NEXT:    br label [[TMP27:%.*]]
; CHECK:       27:
; CHECK-NEXT:    ret i64 257
;
  br label %1

; <label>:
  %.04 = phi ptr [ %src, %0 ], [ %2, %1 ]
  %.013 = phi ptr [ %dst, %0 ], [ %6, %1 ]
  %i.02 = phi i64 [ 0, %0 ], [ %7, %1 ]
  %2 = getelementptr inbounds i16, ptr %.04, i64 1
  %3 = load i16, ptr %.04, align 2
  %4 = zext i16 %3 to i32
  %5 = shl nuw nsw i32 %4, 7
  %6 = getelementptr inbounds i32, ptr %.013, i64 1
  store i32 %5, ptr %.013, align 4
  %7 = add nsw i64 %i.02, 1
  %exitcond = icmp eq i64 %7, 257
  br i1 %exitcond, label %8, label %1

; <label>:
  ret i64 %7
}