llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "br i1|!prof|!llvm.loop|^.*:" --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s --check-prefix=VF4IC1
; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=2 -S %s | FileCheck %s --check-prefix=VF2IC2

; Predicated store where the condition is taken 1 in 8 iterations.
define void @predicated_store(ptr %a, i32 %n) {
; VF4IC1-LABEL: define void @predicated_store(
; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1:![0-9]+]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF1]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF1]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF1]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2:![0-9]+]], !llvm.loop [[LOOP3:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7:![0-9]+]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF1]]
; VF4IC1:  [[IF_THEN]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8:![0-9]+]], !llvm.loop [[LOOP9:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store(
; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1:![0-9]+]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF1]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF1]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF1]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2:![0-9]+]], !llvm.loop [[LOOP3:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7:![0-9]+]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF1]]
; VF2IC2:  [[IF_THEN]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8:![0-9]+]], !llvm.loop [[LOOP9:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %cmp = icmp sgt i32 %val, 0
  br i1 %cmp, label %if.then, label %latch, !prof !1

if.then:
  %add = add i32 %val, 1
  store i32 %add, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

; Predicated store guarded by %c1 (taken 1/2) and %c2 (taken 1/4), so it
; executes 1 in 8 iterations.
define void @predicated_store_sunk_address(ptr %a, ptr %b, i32 %n) {
; VF4IC1-LABEL: define void @predicated_store_sunk_address(
; VF4IC1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_MEMCHECK]]:
; VF4IC1:    br i1 [[DIFF_CHECK:%.*]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF1]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF1]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]], !prof [[PROF1]]
; VF4IC1:  [[PRED_STORE_IF7]]:
; VF4IC1:  [[PRED_STORE_CONTINUE8]]:
; VF4IC1:    br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP11:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[C1:%.*]], label %[[M:.*]], label %[[LATCH:.*]], !prof [[PROF12:![0-9]+]]
; VF4IC1:  [[M]]:
; VF4IC1:    br i1 [[C2:%.*]], label %[[IF:.*]], label %[[LATCH]], !prof [[PROF7]]
; VF4IC1:  [[IF]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP13:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_sunk_address(
; VF2IC2-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_MEMCHECK]]:
; VF2IC2:    br i1 [[DIFF_CHECK:%.*]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF1]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF1]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP22:%.*]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]], !prof [[PROF1]]
; VF2IC2:  [[PRED_STORE_IF8]]:
; VF2IC2:  [[PRED_STORE_CONTINUE9]]:
; VF2IC2:    br i1 [[TMP26:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP11:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[C1:%.*]], label %[[M:.*]], label %[[LATCH:.*]], !prof [[PROF12:![0-9]+]]
; VF2IC2:  [[M]]:
; VF2IC2:    br i1 [[C2:%.*]], label %[[IF:.*]], label %[[LATCH]], !prof [[PROF7]]
; VF2IC2:  [[IF]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP13:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep.a = getelementptr inbounds i32, ptr %a, i32 %iv
  %v = load i32, ptr %gep.a, align 4
  %c1 = icmp sgt i32 %v, 0
  br i1 %c1, label %m, label %latch, !prof !2

m:
  %gep.b = getelementptr inbounds i32, ptr %b, i32 %iv
  %c2 = icmp sgt i32 %v, 100
  br i1 %c2, label %if, label %latch, !prof !3

if:
  store i32 %v, ptr %gep.b, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

; Loop without profile data.
define void @predicated_store_no_profile(ptr %a, i32 %n) {
; VF4IC1-LABEL: define void @predicated_store_no_profile(
; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]]
; VF4IC1:  [[IF_THEN]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_no_profile(
; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]]
; VF2IC2:  [[IF_THEN]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %cmp = icmp sgt i32 %val, 0
  br i1 %cmp, label %if.then, label %latch

if.then:
  %add = add i32 %val, 1
  store i32 %add, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop

exit:
  ret void
}

; Profile data for the loop latch, but not internal loop blocks.
define void @predicated_store_unprofiled_predicate(ptr %a, i32 %n) {
; VF4IC1-LABEL: define void @predicated_store_unprofiled_predicate(
; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP2:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP16:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]]
; VF4IC1:  [[IF_THEN]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP17:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_unprofiled_predicate(
; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP16:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]]
; VF2IC2:  [[IF_THEN]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP17:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %cmp = icmp sgt i32 %val, 0
  br i1 %cmp, label %if.then, label %latch

if.then:
  %add = add i32 %val, 1
  store i32 %add, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

define void @predicated_store_almost_always_taken(ptr %a, i32 %n) {
; VF4IC1-LABEL: define void @predicated_store_almost_always_taken(
; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18:![0-9]+]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF18]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF18]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF18]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP18:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP19:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF20:![0-9]+]]
; VF4IC1:  [[IF_THEN]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP21:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_almost_always_taken(
; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18:![0-9]+]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF18]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF18]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF18]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP19:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF20:![0-9]+]]
; VF2IC2:  [[IF_THEN]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP21:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %cmp = icmp sgt i32 %val, 0
  br i1 %cmp, label %if.then, label %latch, !prof !4

if.then:
  %add = add i32 %val, 1
  store i32 %add, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

; Predicate taken 3 in 5 iterations (60%), which is not of the form 1/N. The
; emitted weights must preserve the 3:2 ratio rather than rounding to 1/2.
define void @predicated_store_non_reciprocal(ptr %a, i32 %n) {
; VF4IC1-LABEL: define void @predicated_store_non_reciprocal(
; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF22:![0-9]+]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF22]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF22]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF22]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP21:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP23:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF24:![0-9]+]]
; VF4IC1:  [[IF_THEN]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP25:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_non_reciprocal(
; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF22:![0-9]+]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF22]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF22]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF22]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP23:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP23:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF24:![0-9]+]]
; VF2IC2:  [[IF_THEN]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP25:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %cmp = icmp sgt i32 %val, 0
  br i1 %cmp, label %if.then, label %latch, !prof !5

if.then:
  %add = add i32 %val, 1
  store i32 %add, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

; Nested predicates, each taken 1 in 100001 iterations, so the block is entered
; with a tiny but nonzero probability.
define void @predicated_store_taken_weight_underflow(ptr %a, i32 %n) {
; VF4IC1-LABEL: define void @predicated_store_taken_weight_underflow(
; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF26:![0-9]+]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF26]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF26]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF26]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP15:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP27:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[C1:%.*]], label %[[IF_1:.*]], label %[[LATCH:.*]], !prof [[PROF28:![0-9]+]]
; VF4IC1:  [[IF_1]]:
; VF4IC1:    br i1 [[C2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH]], !prof [[PROF28]]
; VF4IC1:  [[IF_THEN]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP29:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_taken_weight_underflow(
; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF26:![0-9]+]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF26]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF26]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF26]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP19:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP27:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[C1:%.*]], label %[[IF_1:.*]], label %[[LATCH:.*]], !prof [[PROF28:![0-9]+]]
; VF2IC2:  [[IF_1]]:
; VF2IC2:    br i1 [[C2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH]], !prof [[PROF28]]
; VF2IC2:  [[IF_THEN]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP29:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %c1 = icmp sgt i32 %val, 0
  br i1 %c1, label %if.1, label %latch, !prof !6

if.1:
  %c2 = icmp slt i32 %val, 1000
  br i1 %c2, label %if.then, label %latch, !prof !6

if.then:
  store i32 0, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

define void @predicated_block_with_multiple_ops(ptr noalias %a, ptr noalias %b, i32 %n) {
; VF4IC1-LABEL: define void @predicated_block_with_multiple_ops(
; VF4IC1-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF7]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF7]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF7]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF7]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP22:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP30:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF7]]
; VF4IC1:  [[IF_THEN]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP31:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_block_with_multiple_ops(
; VF2IC2-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF7]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF7]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF7]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP19:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF7]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP24:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP30:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[CMP:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF7]]
; VF2IC2:  [[IF_THEN]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP31:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep.b = getelementptr inbounds i32, ptr %b, i32 %iv
  %val = load i32, ptr %gep.b, align 4
  %cmp = icmp sgt i32 %val, 0
  br i1 %cmp, label %if.then, label %latch, !prof !3

if.then:
  %gep.a = getelementptr inbounds i32, ptr %a, i32 %iv
  %l = load i32, ptr %gep.a, align 4
  %add = add i32 %l, 1
  store i32 %add, ptr %gep.a, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

; %if.then is reached along two distinct paths, so its probability is the sum
; of the contributions of both incoming edges, each scaled by the probability
; of its source block relative to the header:
;   %path.a              1/(1+3) = 1/4,  %path.b 3/4
;   %path.a -> %if.then  1/(1+1) = 1/2  =>  1/4 * 1/2  = 1/8
;   %path.b -> %if.then  1/(1+7) = 1/8  =>  3/4 * 1/8  = 3/32
;   %if.then                            =>  1/8 + 3/32 = 7/32  = {7, 25}
define void @predicated_store_multiple_incoming_edges(ptr %a, i32 %n) {
; VF4IC1-LABEL: define void @predicated_store_multiple_incoming_edges(
; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF32:![0-9]+]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF32]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF32]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF32]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP19:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP33:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[C_1:%.*]], label %[[PATH_A:.*]], label %[[PATH_B:.*]], !prof [[PROF7]]
; VF4IC1:  [[PATH_A]]:
; VF4IC1:    br i1 [[C_2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF12]]
; VF4IC1:  [[PATH_B]]:
; VF4IC1:    br i1 [[C_3:%.*]], label %[[IF_THEN]], label %[[LATCH]], !prof [[PROF1]]
; VF4IC1:  [[IF_THEN]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP34:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_multiple_incoming_edges(
; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF32:![0-9]+]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF32]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP21:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF32]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP24:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF32]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP27:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP33:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[C_1:%.*]], label %[[PATH_A:.*]], label %[[PATH_B:.*]], !prof [[PROF7]]
; VF2IC2:  [[PATH_A]]:
; VF2IC2:    br i1 [[C_2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF12]]
; VF2IC2:  [[PATH_B]]:
; VF2IC2:    br i1 [[C_3:%.*]], label %[[IF_THEN]], label %[[LATCH]], !prof [[PROF1]]
; VF2IC2:  [[IF_THEN]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP34:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %c.1 = icmp sgt i32 %val, 0
  br i1 %c.1, label %path.a, label %path.b, !prof !3

path.a:
  %c.2 = icmp sgt i32 %val, 100
  br i1 %c.2, label %if.then, label %latch, !prof !2

path.b:
  %c.3 = icmp slt i32 %val, -100
  br i1 %c.3, label %if.then, label %latch, !prof !1

if.then:
  store i32 0, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

; As above, but the branch of %path.b carries no weights. The probability of
; one incoming edge of %if.then is unknown.
define void @predicated_store_multiple_incoming_edges_one_unprofiled(ptr %a, i32 %n) {
; VF4IC1-LABEL: define void @predicated_store_multiple_incoming_edges_one_unprofiled(
; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP8:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP19:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP35:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[C_1:%.*]], label %[[PATH_A:.*]], label %[[PATH_B:.*]], !prof [[PROF7]]
; VF4IC1:  [[PATH_A]]:
; VF4IC1:    br i1 [[C_2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF12]]
; VF4IC1:  [[PATH_B]]:
; VF4IC1:    br i1 [[C_3:%.*]], label %[[IF_THEN]], label %[[LATCH]]
; VF4IC1:  [[IF_THEN]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP36:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_multiple_incoming_edges_one_unprofiled(
; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP20:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP23:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP27:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP35:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[C_1:%.*]], label %[[PATH_A:.*]], label %[[PATH_B:.*]], !prof [[PROF7]]
; VF2IC2:  [[PATH_A]]:
; VF2IC2:    br i1 [[C_2:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF12]]
; VF2IC2:  [[PATH_B]]:
; VF2IC2:    br i1 [[C_3:%.*]], label %[[IF_THEN]], label %[[LATCH]]
; VF2IC2:  [[IF_THEN]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP36:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %c.1 = icmp sgt i32 %val, 0
  br i1 %c.1, label %path.a, label %path.b, !prof !3

path.a:
  %c.2 = icmp sgt i32 %val, 100
  br i1 %c.2, label %if.then, label %latch, !prof !2

path.b:
  %c.3 = icmp slt i32 %val, -100
  br i1 %c.3, label %if.then, label %latch

if.then:
  store i32 0, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

; Two cases of the switch branch to %if.then, so the weights of both edges
; contribute to its probability. With a total weight of 4+1+2+1 = 8,
;   %if.then (cases 0 and 1)  (1 + 2)/8 = 3/8  = {3, 5}
;   %other   (case 2)               1/8       = {1, 7}
define void @predicated_store_switch_multiple_cases(ptr %a, i32 %n) {
; VF4IC1-LABEL: define void @predicated_store_switch_multiple_cases(
; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF1]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP14:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF1]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF1]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP16:%.*]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]], !prof [[PROF37:![0-9]+]]
; VF4IC1:  [[PRED_STORE_IF7]]:
; VF4IC1:  [[PRED_STORE_CONTINUE8]]:
; VF4IC1:    br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]], !prof [[PROF37]]
; VF4IC1:  [[PRED_STORE_IF9]]:
; VF4IC1:  [[PRED_STORE_CONTINUE10]]:
; VF4IC1:    br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]], !prof [[PROF37]]
; VF4IC1:  [[PRED_STORE_IF11]]:
; VF4IC1:  [[PRED_STORE_CONTINUE12]]:
; VF4IC1:    br i1 [[TMP19:%.*]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]], !prof [[PROF37]]
; VF4IC1:  [[PRED_STORE_IF13]]:
; VF4IC1:  [[PRED_STORE_CONTINUE14]]:
; VF4IC1:    br i1 [[TMP20:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP38:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    ], !prof [[PROF39:![0-9]+]]
; VF4IC1:  [[IF_THEN:.*:]]
; VF4IC1:  [[OTHER:.*:]]
; VF4IC1:  [[LATCH:.*:]]
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP40:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_switch_multiple_cases(
; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP17:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF1]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP18:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF1]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP19:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF1]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP20:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF1]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP21:%.*]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]], !prof [[PROF37:![0-9]+]]
; VF2IC2:  [[PRED_STORE_IF8]]:
; VF2IC2:  [[PRED_STORE_CONTINUE9]]:
; VF2IC2:    br i1 [[TMP22:%.*]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]], !prof [[PROF37]]
; VF2IC2:  [[PRED_STORE_IF10]]:
; VF2IC2:  [[PRED_STORE_CONTINUE11]]:
; VF2IC2:    br i1 [[TMP23:%.*]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]], !prof [[PROF37]]
; VF2IC2:  [[PRED_STORE_IF12]]:
; VF2IC2:  [[PRED_STORE_CONTINUE13]]:
; VF2IC2:    br i1 [[TMP24:%.*]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15:.*]], !prof [[PROF37]]
; VF2IC2:  [[PRED_STORE_IF14]]:
; VF2IC2:  [[PRED_STORE_CONTINUE15]]:
; VF2IC2:    br i1 [[TMP25:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP38:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    ], !prof [[PROF39:![0-9]+]]
; VF2IC2:  [[IF_THEN:.*:]]
; VF2IC2:  [[OTHER:.*:]]
; VF2IC2:  [[LATCH:.*:]]
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP40:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  switch i32 %val, label %latch [
  i32 0, label %if.then
  i32 1, label %if.then
  i32 2, label %other
  ], !prof !7

if.then:
  store i32 0, ptr %gep, align 4
  br label %latch

other:
  store i32 1, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

; Two predicated stores guarded by the same condition, but with inconsistent
; branch weights: the first branch is almost always taken, the second almost
; never.
define void @merged_replicate_regions_inconsistent_weights(ptr noalias %a, ptr noalias %b, i32 %n) {
; VF4IC1-LABEL: define void @merged_replicate_regions_inconsistent_weights(
; VF4IC1-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF18]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF18]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF18]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP17:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP41:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[CMP:%.*]], label %[[IF_THEN_1:.*]], label %[[MERGE:.*]], !prof [[PROF20]]
; VF4IC1:  [[IF_THEN_1]]:
; VF4IC1:  [[MERGE]]:
; VF4IC1:    br i1 [[CMP]], label %[[IF_THEN_2:.*]], label %[[LATCH:.*]], !prof [[PROF42:![0-9]+]]
; VF4IC1:  [[IF_THEN_2]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP43:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @merged_replicate_regions_inconsistent_weights(
; VF2IC2-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF18]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF18]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF18]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF18]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP19:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP41:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[CMP:%.*]], label %[[IF_THEN_1:.*]], label %[[MERGE:.*]], !prof [[PROF20]]
; VF2IC2:  [[IF_THEN_1]]:
; VF2IC2:  [[MERGE]]:
; VF2IC2:    br i1 [[CMP]], label %[[IF_THEN_2:.*]], label %[[LATCH:.*]], !prof [[PROF42:![0-9]+]]
; VF2IC2:  [[IF_THEN_2]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP43:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep.a = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep.a, align 4
  %cmp = icmp sgt i32 %val, 0
  br i1 %cmp, label %if.then.1, label %merge, !prof !4

if.then.1:
  store i32 0, ptr %gep.a, align 4
  br label %merge

merge:
  br i1 %cmp, label %if.then.2, label %latch, !prof !0

if.then.2:
  %gep.b = getelementptr inbounds i32, ptr %b, i32 %iv
  store i32 0, ptr %gep.b, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

; Predicated store where the condition is taken with a probability just below
; one. Rounding the probability must not push it to exactly one, which would
; drop the branch weights entirely, as a block that always executes needs none.
define void @predicated_store_taken_weight_rounds_to_one(ptr %a, i32 %n) {
; VF4IC1-LABEL: define void @predicated_store_taken_weight_rounds_to_one(
; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF44:![0-9]+]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF44]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF44]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF44]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP13:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP45:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[C:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF46:![0-9]+]]
; VF4IC1:  [[IF_THEN]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP47:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_taken_weight_rounds_to_one(
; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF44:![0-9]+]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF44]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF44]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF44]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP15:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP45:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[C:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF46:![0-9]+]]
; VF2IC2:  [[IF_THEN]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP47:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %c = icmp sgt i32 %val, 0
  br i1 %c, label %if.then, label %latch, !prof !8

if.then:
  store i32 0, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

; Two predicated stores guarded by the same condition. The first branch is
; almost always taken (its other edge has weight 0), while the second is
; rarely taken. The merged region fires whenever either original region did,
; i.e. almost always, so it must not keep the second region's low frequency.
define void @merged_replicate_regions_first_always_taken(ptr noalias %a, ptr noalias %b, i32 %n) {
; VF4IC1-LABEL: define void @merged_replicate_regions_first_always_taken(
; VF4IC1-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF44]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF44]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF44]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP13:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF44]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP17:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP48:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[CMP:%.*]], label %[[IF_THEN_1:.*]], label %[[MERGE:.*]], !prof [[PROF8]]
; VF4IC1:  [[IF_THEN_1]]:
; VF4IC1:  [[MERGE]]:
; VF4IC1:    br i1 [[CMP]], label %[[IF_THEN_2:.*]], label %[[LATCH:.*]], !prof [[PROF42]]
; VF4IC1:  [[IF_THEN_2]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP49:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @merged_replicate_regions_first_always_taken(
; VF2IC2-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF44]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF44]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP11:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF44]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP15:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF44]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP19:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP48:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[CMP:%.*]], label %[[IF_THEN_1:.*]], label %[[MERGE:.*]], !prof [[PROF8]]
; VF2IC2:  [[IF_THEN_1]]:
; VF2IC2:  [[MERGE]]:
; VF2IC2:    br i1 [[CMP]], label %[[IF_THEN_2:.*]], label %[[LATCH:.*]], !prof [[PROF42]]
; VF2IC2:  [[IF_THEN_2]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP49:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep.a = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep.a, align 4
  %cmp = icmp sgt i32 %val, 0
  br i1 %cmp, label %if.then.1, label %merge, !prof !9

if.then.1:
  store i32 0, ptr %gep.a, align 4
  br label %merge

merge:
  br i1 %cmp, label %if.then.2, label %latch, !prof !0

if.then.2:
  %gep.b = getelementptr inbounds i32, ptr %b, i32 %iv
  store i32 0, ptr %gep.b, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

; Predicated store where the condition has a zero taken weight. As in
; BlockFrequencyInfo, a zero weight marks the edge as cold, not as never taken,
; so the replicate region branches must still get (very unlikely) weights.
define void @predicated_store_zero_taken_weight(ptr %a, i32 %n) {
; VF4IC1-LABEL: define void @predicated_store_zero_taken_weight(
; VF4IC1-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF4IC1:  [[ENTRY:.*:]]
; VF4IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF4IC1:  [[VECTOR_PH]]:
; VF4IC1:  [[VECTOR_BODY:.*]]:
; VF4IC1:    br i1 [[TMP3:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF26]]
; VF4IC1:  [[PRED_STORE_IF]]:
; VF4IC1:  [[PRED_STORE_CONTINUE]]:
; VF4IC1:    br i1 [[TMP4:%.*]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF26]]
; VF4IC1:  [[PRED_STORE_IF1]]:
; VF4IC1:  [[PRED_STORE_CONTINUE2]]:
; VF4IC1:    br i1 [[TMP7:%.*]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF26]]
; VF4IC1:  [[PRED_STORE_IF3]]:
; VF4IC1:  [[PRED_STORE_CONTINUE4]]:
; VF4IC1:    br i1 [[TMP10:%.*]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]], !prof [[PROF26]]
; VF4IC1:  [[PRED_STORE_IF5]]:
; VF4IC1:  [[PRED_STORE_CONTINUE6]]:
; VF4IC1:    br i1 [[TMP13:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP50:![0-9]+]]
; VF4IC1:  [[MIDDLE_BLOCK]]:
; VF4IC1:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF4IC1:  [[SCALAR_PH]]:
; VF4IC1:  [[LOOP:.*]]:
; VF4IC1:    br i1 [[C:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF51:![0-9]+]]
; VF4IC1:  [[IF_THEN]]:
; VF4IC1:  [[LATCH]]:
; VF4IC1:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP52:![0-9]+]]
; VF4IC1:  [[EXIT]]:
;
; VF2IC2-LABEL: define void @predicated_store_zero_taken_weight(
; VF2IC2-SAME: ptr [[A:%.*]], i32 [[N:%.*]]) {
; VF2IC2:  [[ENTRY:.*:]]
; VF2IC2:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
; VF2IC2:  [[VECTOR_PH]]:
; VF2IC2:  [[VECTOR_BODY:.*]]:
; VF2IC2:    br i1 [[TMP5:%.*]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF26]]
; VF2IC2:  [[PRED_STORE_IF]]:
; VF2IC2:  [[PRED_STORE_CONTINUE]]:
; VF2IC2:    br i1 [[TMP6:%.*]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]], !prof [[PROF26]]
; VF2IC2:  [[PRED_STORE_IF2]]:
; VF2IC2:  [[PRED_STORE_CONTINUE3]]:
; VF2IC2:    br i1 [[TMP9:%.*]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]], !prof [[PROF26]]
; VF2IC2:  [[PRED_STORE_IF4]]:
; VF2IC2:  [[PRED_STORE_CONTINUE5]]:
; VF2IC2:    br i1 [[TMP12:%.*]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]], !prof [[PROF26]]
; VF2IC2:  [[PRED_STORE_IF6]]:
; VF2IC2:  [[PRED_STORE_CONTINUE7]]:
; VF2IC2:    br i1 [[TMP15:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF2]], !llvm.loop [[LOOP50:![0-9]+]]
; VF2IC2:  [[MIDDLE_BLOCK]]:
; VF2IC2:    br i1 [[CMP_N:%.*]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !prof [[PROF7]]
; VF2IC2:  [[SCALAR_PH]]:
; VF2IC2:  [[LOOP:.*]]:
; VF2IC2:    br i1 [[C:%.*]], label %[[IF_THEN:.*]], label %[[LATCH:.*]], !prof [[PROF51:![0-9]+]]
; VF2IC2:  [[IF_THEN]]:
; VF2IC2:  [[LATCH]]:
; VF2IC2:    br i1 [[EXITCOND:%.*]], label %[[EXIT]], label %[[LOOP]], !prof [[PROF8]], !llvm.loop [[LOOP52:![0-9]+]]
; VF2IC2:  [[EXIT]]:
;
entry:
  br label %loop

loop:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
  %gep = getelementptr inbounds i32, ptr %a, i32 %iv
  %val = load i32, ptr %gep, align 4
  %c = icmp sgt i32 %val, 0
  br i1 %c, label %if.then, label %latch, !prof !10

if.then:
  store i32 0, ptr %gep, align 4
  br label %latch

latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %exitcond = icmp eq i32 %iv.next, %n
  br i1 %exitcond, label %exit, label %loop, !prof !0

exit:
  ret void
}

!0 = !{!"branch_weights", i32 1, i32 1000}
!1 = !{!"branch_weights", i32 1, i32 7}
!2 = !{!"branch_weights", i32 1, i32 1}
!3 = !{!"branch_weights", i32 1, i32 3}
!4 = !{!"branch_weights", i32 1000, i32 1}
!5 = !{!"branch_weights", i32 3, i32 2}
!6 = !{!"branch_weights", i32 1, i32 100000}
!7 = !{!"branch_weights", i32 4, i32 1, i32 2, i32 1}
!8 = !{!"branch_weights", i32 4294967295, i32 1}
!9 = !{!"branch_weights", i32 1, i32 0}
!10 = !{!"branch_weights", i32 0, i32 1000}
;.
; VF4IC1: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
; VF4IC1: [[PROF1]] = !{!"branch_weights", i32 1, i32 7}
; VF4IC1: [[PROF2]] = !{!"branch_weights", i32 1, i32 249}
; VF4IC1: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
; VF4IC1: [[META4]] = !{!"llvm.loop.isvectorized", i32 1}
; VF4IC1: [[META5]] = !{!"llvm.loop.unroll.runtime.disable"}
; VF4IC1: [[META6]] = !{!"llvm.loop.estimated_trip_count", i32 250}
; VF4IC1: [[PROF7]] = !{!"branch_weights", i32 1, i32 3}
; VF4IC1: [[PROF8]] = !{!"branch_weights", i32 1, i32 0}
; VF4IC1: [[LOOP9]] = distinct !{[[LOOP9]], [[META5]], [[META4]], [[META10:![0-9]+]]}
; VF4IC1: [[META10]] = !{!"llvm.loop.estimated_trip_count", i32 1}
; VF4IC1: [[LOOP11]] = distinct !{[[LOOP11]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[PROF12]] = !{!"branch_weights", i32 1, i32 1}
; VF4IC1: [[LOOP13]] = distinct !{[[LOOP13]], [[META4]], [[META10]]}
; VF4IC1: [[LOOP14]] = distinct !{[[LOOP14]], [[META4]], [[META5]]}
; VF4IC1: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]], [[META4]]}
; VF4IC1: [[LOOP16]] = distinct !{[[LOOP16]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]], [[META4]], [[META10]]}
; VF4IC1: [[PROF18]] = !{!"branch_weights", i32 1072669155, i32 1072669}
; VF4IC1: [[LOOP19]] = distinct !{[[LOOP19]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[PROF20]] = !{!"branch_weights", i32 1000, i32 1}
; VF4IC1: [[LOOP21]] = distinct !{[[LOOP21]], [[META5]], [[META4]], [[META10]]}
; VF4IC1: [[PROF22]] = !{!"branch_weights", i32 1288490189, i32 858993459}
; VF4IC1: [[LOOP23]] = distinct !{[[LOOP23]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[PROF24]] = !{!"branch_weights", i32 3, i32 2}
; VF4IC1: [[LOOP25]] = distinct !{[[LOOP25]], [[META5]], [[META4]], [[META10]]}
; VF4IC1: [[PROF26]] = !{!"branch_weights", i32 1, i32 2147483647}
; VF4IC1: [[LOOP27]] = distinct !{[[LOOP27]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[PROF28]] = !{!"branch_weights", i32 1, i32 100000}
; VF4IC1: [[LOOP29]] = distinct !{[[LOOP29]], [[META5]], [[META4]], [[META10]]}
; VF4IC1: [[LOOP30]] = distinct !{[[LOOP30]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[LOOP31]] = distinct !{[[LOOP31]], [[META5]], [[META4]], [[META10]]}
; VF4IC1: [[PROF32]] = !{!"branch_weights", i32 7, i32 25}
; VF4IC1: [[LOOP33]] = distinct !{[[LOOP33]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[LOOP34]] = distinct !{[[LOOP34]], [[META5]], [[META4]], [[META10]]}
; VF4IC1: [[LOOP35]] = distinct !{[[LOOP35]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[LOOP36]] = distinct !{[[LOOP36]], [[META5]], [[META4]], [[META10]]}
; VF4IC1: [[PROF37]] = !{!"branch_weights", i32 3, i32 5}
; VF4IC1: [[LOOP38]] = distinct !{[[LOOP38]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[PROF39]] = !{!"branch_weights", i32 4, i32 1, i32 2, i32 1}
; VF4IC1: [[LOOP40]] = distinct !{[[LOOP40]], [[META5]], [[META4]], [[META10]]}
; VF4IC1: [[LOOP41]] = distinct !{[[LOOP41]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[PROF42]] = !{!"branch_weights", i32 1, i32 1000}
; VF4IC1: [[LOOP43]] = distinct !{[[LOOP43]], [[META5]], [[META4]], [[META10]]}
; VF4IC1: [[PROF44]] = !{!"branch_weights", i32 2147483647, i32 1}
; VF4IC1: [[LOOP45]] = distinct !{[[LOOP45]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[PROF46]] = !{!"branch_weights", i32 -1, i32 1}
; VF4IC1: [[LOOP47]] = distinct !{[[LOOP47]], [[META5]], [[META4]], [[META10]]}
; VF4IC1: [[LOOP48]] = distinct !{[[LOOP48]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[LOOP49]] = distinct !{[[LOOP49]], [[META5]], [[META4]], [[META10]]}
; VF4IC1: [[LOOP50]] = distinct !{[[LOOP50]], [[META4]], [[META5]], [[META6]]}
; VF4IC1: [[PROF51]] = !{!"branch_weights", i32 0, i32 1000}
; VF4IC1: [[LOOP52]] = distinct !{[[LOOP52]], [[META5]], [[META4]], [[META10]]}
;.
; VF2IC2: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
; VF2IC2: [[PROF1]] = !{!"branch_weights", i32 1, i32 7}
; VF2IC2: [[PROF2]] = !{!"branch_weights", i32 1, i32 249}
; VF2IC2: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
; VF2IC2: [[META4]] = !{!"llvm.loop.isvectorized", i32 1}
; VF2IC2: [[META5]] = !{!"llvm.loop.unroll.runtime.disable"}
; VF2IC2: [[META6]] = !{!"llvm.loop.estimated_trip_count", i32 250}
; VF2IC2: [[PROF7]] = !{!"branch_weights", i32 1, i32 3}
; VF2IC2: [[PROF8]] = !{!"branch_weights", i32 1, i32 0}
; VF2IC2: [[LOOP9]] = distinct !{[[LOOP9]], [[META5]], [[META4]], [[META10:![0-9]+]]}
; VF2IC2: [[META10]] = !{!"llvm.loop.estimated_trip_count", i32 1}
; VF2IC2: [[LOOP11]] = distinct !{[[LOOP11]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[PROF12]] = !{!"branch_weights", i32 1, i32 1}
; VF2IC2: [[LOOP13]] = distinct !{[[LOOP13]], [[META4]], [[META10]]}
; VF2IC2: [[LOOP14]] = distinct !{[[LOOP14]], [[META4]], [[META5]]}
; VF2IC2: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]], [[META4]]}
; VF2IC2: [[LOOP16]] = distinct !{[[LOOP16]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]], [[META4]], [[META10]]}
; VF2IC2: [[PROF18]] = !{!"branch_weights", i32 1072669155, i32 1072669}
; VF2IC2: [[LOOP19]] = distinct !{[[LOOP19]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[PROF20]] = !{!"branch_weights", i32 1000, i32 1}
; VF2IC2: [[LOOP21]] = distinct !{[[LOOP21]], [[META5]], [[META4]], [[META10]]}
; VF2IC2: [[PROF22]] = !{!"branch_weights", i32 1288490189, i32 858993459}
; VF2IC2: [[LOOP23]] = distinct !{[[LOOP23]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[PROF24]] = !{!"branch_weights", i32 3, i32 2}
; VF2IC2: [[LOOP25]] = distinct !{[[LOOP25]], [[META5]], [[META4]], [[META10]]}
; VF2IC2: [[PROF26]] = !{!"branch_weights", i32 1, i32 2147483647}
; VF2IC2: [[LOOP27]] = distinct !{[[LOOP27]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[PROF28]] = !{!"branch_weights", i32 1, i32 100000}
; VF2IC2: [[LOOP29]] = distinct !{[[LOOP29]], [[META5]], [[META4]], [[META10]]}
; VF2IC2: [[LOOP30]] = distinct !{[[LOOP30]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[LOOP31]] = distinct !{[[LOOP31]], [[META5]], [[META4]], [[META10]]}
; VF2IC2: [[PROF32]] = !{!"branch_weights", i32 7, i32 25}
; VF2IC2: [[LOOP33]] = distinct !{[[LOOP33]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[LOOP34]] = distinct !{[[LOOP34]], [[META5]], [[META4]], [[META10]]}
; VF2IC2: [[LOOP35]] = distinct !{[[LOOP35]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[LOOP36]] = distinct !{[[LOOP36]], [[META5]], [[META4]], [[META10]]}
; VF2IC2: [[PROF37]] = !{!"branch_weights", i32 3, i32 5}
; VF2IC2: [[LOOP38]] = distinct !{[[LOOP38]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[PROF39]] = !{!"branch_weights", i32 4, i32 1, i32 2, i32 1}
; VF2IC2: [[LOOP40]] = distinct !{[[LOOP40]], [[META5]], [[META4]], [[META10]]}
; VF2IC2: [[LOOP41]] = distinct !{[[LOOP41]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[PROF42]] = !{!"branch_weights", i32 1, i32 1000}
; VF2IC2: [[LOOP43]] = distinct !{[[LOOP43]], [[META5]], [[META4]], [[META10]]}
; VF2IC2: [[PROF44]] = !{!"branch_weights", i32 2147483647, i32 1}
; VF2IC2: [[LOOP45]] = distinct !{[[LOOP45]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[PROF46]] = !{!"branch_weights", i32 -1, i32 1}
; VF2IC2: [[LOOP47]] = distinct !{[[LOOP47]], [[META5]], [[META4]], [[META10]]}
; VF2IC2: [[LOOP48]] = distinct !{[[LOOP48]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[LOOP49]] = distinct !{[[LOOP49]], [[META5]], [[META4]], [[META10]]}
; VF2IC2: [[LOOP50]] = distinct !{[[LOOP50]], [[META4]], [[META5]], [[META6]]}
; VF2IC2: [[PROF51]] = !{!"branch_weights", i32 0, i32 1000}
; VF2IC2: [[LOOP52]] = distinct !{[[LOOP52]], [[META5]], [[META4]], [[META10]]}
;.