llvm/test/Transforms/LoopVectorize/AArch64/alias-mask-uniforms.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
; RUN: opt -S -force-partial-aliasing-vectorization -force-target-supports-masked-memory-ops -tail-folding-policy=must-fold-tail -force-vector-width=4 -passes=loop-vectorize %s | FileCheck %s

; Tests `%div = sdiv i64 %iv, 64` (and its use for a load) is not considered
; to be uniform with partial alias masking.
define void @vf_dependent_uniform(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-LABEL: define void @vf_dependent_uniform(
; CHECK-SAME: ptr noalias [[P:%.*]], ptr [[P_OUT:%.*]], ptr [[P_IN:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[P_OUT1:%.*]] = ptrtoaddr ptr [[P_OUT]] to i64
; CHECK-NEXT:    [[P_IN2:%.*]] = ptrtoaddr ptr [[P_IN]] to i64
; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT:    br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK:       [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT:    [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[P_IN2]], i64 [[P_OUT1]], i64 8)
; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i64>
; CHECK-NEXT:    [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
; CHECK-NEXT:    [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 -1, [[SMAX]]
; CHECK-NEXT:    [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[SMAX]], 1
; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[SMAX]], [[TMP7]]
; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT4]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE10:.*]] ]
; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE10]] ]
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT4]]
; CHECK-NEXT:    [[MASK:%.*]] = and <4 x i1> [[TMP8]], [[ALIAS_MASK]]
; CHECK-NEXT:    [[TMP10:%.*]] = sdiv <4 x i64> [[VEC_IND]], splat (i64 64)
; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[MASK]], i64 0
; CHECK-NEXT:    br i1 [[TMP11]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK:       [[PRED_LOAD_IF]]:
; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[TMP10]], i64 0
; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP12]]
; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP13]], align 8
; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> poison, i64 [[TMP14]], i64 0
; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
; CHECK:       [[PRED_LOAD_CONTINUE]]:
; CHECK-NEXT:    [[TMP16:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP15]], %[[PRED_LOAD_IF]] ]
; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i1> [[MASK]], i64 1
; CHECK-NEXT:    br i1 [[TMP17]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
; CHECK:       [[PRED_LOAD_IF5]]:
; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i64> [[TMP10]], i64 1
; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP18]]
; CHECK-NEXT:    [[TMP20:%.*]] = load i64, ptr [[TMP19]], align 8
; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP20]], i64 1
; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE6]]
; CHECK:       [[PRED_LOAD_CONTINUE6]]:
; CHECK-NEXT:    [[TMP22:%.*]] = phi <4 x i64> [ [[TMP16]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP21]], %[[PRED_LOAD_IF5]] ]
; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <4 x i1> [[MASK]], i64 2
; CHECK-NEXT:    br i1 [[TMP23]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]
; CHECK:       [[PRED_LOAD_IF7]]:
; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x i64> [[TMP10]], i64 2
; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP24]]
; CHECK-NEXT:    [[TMP26:%.*]] = load i64, ptr [[TMP25]], align 8
; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <4 x i64> [[TMP22]], i64 [[TMP26]], i64 2
; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE8]]
; CHECK:       [[PRED_LOAD_CONTINUE8]]:
; CHECK-NEXT:    [[TMP28:%.*]] = phi <4 x i64> [ [[TMP22]], %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP27]], %[[PRED_LOAD_IF7]] ]
; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x i1> [[MASK]], i64 3
; CHECK-NEXT:    br i1 [[TMP29]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10]]
; CHECK:       [[PRED_LOAD_IF9]]:
; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <4 x i64> [[TMP10]], i64 3
; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP30]]
; CHECK-NEXT:    [[TMP32:%.*]] = load i64, ptr [[TMP31]], align 8
; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <4 x i64> [[TMP28]], i64 [[TMP32]], i64 3
; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE10]]
; CHECK:       [[PRED_LOAD_CONTINUE10]]:
; CHECK-NEXT:    [[TMP34:%.*]] = phi <4 x i64> [ [[TMP28]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP33]], %[[PRED_LOAD_IF9]] ]
; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i64, ptr [[P_IN]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP35]], <4 x i1> [[MASK]], <4 x i64> poison)
; CHECK-NEXT:    [[TMP36:%.*]] = add <4 x i64> [[TMP34]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP36]], ptr align 8 [[TMP37]], <4 x i1> [[MASK]])
; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br [[EXIT:label %.*]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %div = sdiv i64 %iv, 64
  %gep = getelementptr inbounds i64, ptr %p, i64 %div
  %ld = load i64, ptr %gep, align 8
  %gep2 = getelementptr inbounds i64, ptr %p.in, i64 %iv
  %ld2 = load i64, ptr %gep2, align 8
  %val = add i64 %ld, %ld2
  %store.gep = getelementptr i64, ptr %p.out, i64 %iv
  store i64 %val, ptr %store.gep, align 8
  %iv.next = add nsw i64 %iv, 1
  %exitcond = icmp slt i64 %iv.next, %n
  br i1 %exitcond, label %loop, label %exit
exit:
  ret void
}

define void @uniform_load(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-LABEL: define void @uniform_load(
; CHECK-SAME: ptr noalias [[P:%.*]], ptr [[P_OUT:%.*]], ptr [[P_IN:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[P_OUT1:%.*]] = ptrtoaddr ptr [[P_OUT]] to i64
; CHECK-NEXT:    [[P_IN2:%.*]] = ptrtoaddr ptr [[P_IN]] to i64
; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT:    br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK:       [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT:    [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[P_IN2]], i64 [[P_OUT1]], i64 8)
; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i64>
; CHECK-NEXT:    [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
; CHECK-NEXT:    [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 -1, [[SMAX]]
; CHECK-NEXT:    [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[SMAX]], 1
; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[SMAX]], [[TMP7]]
; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[MASK:%.*]] = and <4 x i1> [[TMP8]], [[ALIAS_MASK]]
; CHECK-NEXT:    [[TMP22:%.*]] = load i64, ptr [[P]], align 8
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i64> poison, i64 [[TMP22]], i64 0
; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT5]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[P_IN]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP18]], <4 x i1> [[MASK]], <4 x i64> poison)
; CHECK-NEXT:    [[TMP19:%.*]] = add <4 x i64> [[TMP17]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP19]], ptr align 8 [[TMP20]], <4 x i1> [[MASK]])
; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT1]]
; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br [[EXIT:label %.*]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %ld = load i64, ptr %p, align 8
  %gep = getelementptr inbounds i64, ptr %p.in, i64 %iv
  %ld2 = load i64, ptr %gep, align 8
  %val = add i64 %ld, %ld2
  %store.gep = getelementptr i64, ptr %p.out, i64 %iv
  store i64 %val, ptr %store.gep, align 8
  %iv.next = add nsw i64 %iv, 1
  %exitcond = icmp slt i64 %iv.next, %n
  br i1 %exitcond, label %loop, label %exit
exit:
  ret void
}

define void @uniform_store(ptr noalias %p, ptr %p.out, ptr %p.in, i64 %n) {
; CHECK-LABEL: define void @uniform_store(
; CHECK-SAME: ptr noalias [[P:%.*]], ptr [[P_OUT:%.*]], ptr [[P_IN:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    [[P_OUT1:%.*]] = ptrtoaddr ptr [[P_OUT]] to i64
; CHECK-NEXT:    [[P_IN2:%.*]] = ptrtoaddr ptr [[P_IN]] to i64
; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; CHECK-NEXT:    br label %[[VECTOR_CLAMPED_VF_CHECK:.*]]
; CHECK:       [[VECTOR_CLAMPED_VF_CHECK]]:
; CHECK-NEXT:    [[ALIAS_MASK:%.*]] = call <4 x i1> @llvm.loop.dependence.war.mask.v4i1.i64(i64 [[P_IN2]], i64 [[P_OUT1]], i64 8)
; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i1> [[ALIAS_MASK]] to <4 x i64>
; CHECK-NEXT:    [[NUM_ACTIVE_LANES:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
; CHECK-NEXT:    [[VF_IS_SCALAR:%.*]] = icmp ule i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 -1, [[SMAX]]
; CHECK-NEXT:    [[VF_STEP_OVERFLOW:%.*]] = icmp ult i64 [[TMP5]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT:    [[TMP6:%.*]] = or i1 [[VF_IS_SCALAR]], [[VF_STEP_OVERFLOW]]
; CHECK-NEXT:    br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[SMAX]], 1
; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[NUM_ACTIVE_LANES]], 1
; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[SMAX]], [[TMP7]]
; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[NUM_ACTIVE_LANES]], i64 0
; CHECK-NEXT:    [[BROADCAST_SPLAT1:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE10:.*]] ]
; CHECK-NEXT:    [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE10]] ]
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT:    [[MASK:%.*]] = and <4 x i1> [[TMP8]], [[ALIAS_MASK]]
; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[P_IN]], i64 [[INDEX]]
; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP10]], <4 x i1> [[MASK]], <4 x i64> poison)
; CHECK-NEXT:    [[TMP11:%.*]] = add <4 x i64> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i1> [[MASK]], i64 0
; CHECK-NEXT:    br i1 [[TMP12]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK:       [[PRED_STORE_IF]]:
; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i64> [[TMP11]], i64 0
; CHECK-NEXT:    store i64 [[TMP13]], ptr [[P]], align 8
; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
; CHECK:       [[PRED_STORE_CONTINUE]]:
; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[MASK]], i64 1
; CHECK-NEXT:    br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; CHECK:       [[PRED_STORE_IF5]]:
; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i64> [[TMP11]], i64 1
; CHECK-NEXT:    store i64 [[TMP15]], ptr [[P]], align 8
; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
; CHECK:       [[PRED_STORE_CONTINUE6]]:
; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[MASK]], i64 2
; CHECK-NEXT:    br i1 [[TMP16]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
; CHECK:       [[PRED_STORE_IF7]]:
; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i64> [[TMP11]], i64 2
; CHECK-NEXT:    store i64 [[TMP17]], ptr [[P]], align 8
; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE8]]
; CHECK:       [[PRED_STORE_CONTINUE8]]:
; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i1> [[MASK]], i64 3
; CHECK-NEXT:    br i1 [[TMP18]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10]]
; CHECK:       [[PRED_STORE_IF9]]:
; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <4 x i64> [[TMP11]], i64 3
; CHECK-NEXT:    store i64 [[TMP19]], ptr [[P]], align 8
; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE10]]
; CHECK:       [[PRED_STORE_CONTINUE10]]:
; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0(<4 x i64> [[TMP11]], ptr align 8 [[TMP20]], <4 x i1> [[MASK]])
; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[NUM_ACTIVE_LANES]]
; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT1]]
; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br [[EXIT:label %.*]]
; CHECK:       [[SCALAR_PH]]:
;
entry:
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
  %gep = getelementptr inbounds i64, ptr %p.in, i64 %iv
  %ld = load i64, ptr %gep, align 8
  %val = add i64 %ld, %ld
  store i64 %val, ptr %p, align 8
  %store.gep = getelementptr i64, ptr %p.out, i64 %iv
  store i64 %val, ptr %store.gep, align 8
  %iv.next = add nsw i64 %iv, 1
  %exitcond = icmp slt i64 %iv.next, %n
  br i1 %exitcond, label %loop, label %exit
exit:
  ret void
}