blob: f29a9fbfdf9946a943f49cc4dab0873111ae9112 [file]
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^middle.block:" --version 6
; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-interleave=4 -aarch64-enable-subreg-liveness-tracking -sve-tail-folding=disabled %s | FileCheck %s
; RUN: opt -S -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -force-vector-interleave=2 -aarch64-enable-subreg-liveness-tracking -tail-folding-policy=must-fold-tail %s | FileCheck %s --check-prefix=CHECK-UF2-TF
target triple = "aarch64-unknown-linux-gnu"
; Note: `noalias` was not applied to the pointer arguments to test !noalias metadata on the loads/stores.
define void @mixed_i64_i32_accesses(ptr %x, ptr %y, i64 %n) {
; CHECK-LABEL: define void @mixed_i64_i32_accesses(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ITER_CHECK:.*:]]
; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[TMP33:%.*]] = shl i64 [[TMP5]], 3
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[X]], i64 [[TMP33]]
; CHECK-NEXT: [[TMP34:%.*]] = shl i64 [[TMP5]], 2
; CHECK-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[Y]], i64 [[TMP34]]
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[X]], [[TMP35]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[Y]], [[TMP2]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], [[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK1:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK1]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP4:%.*]] = load <vscale x 16 x i64>, ptr [[TMP3]], align 8, !alias.scope [[META3:![0-9]+]], !noalias [[META6:![0-9]+]]
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 0)
; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 4)
; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 8)
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 12)
; CHECK-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[TMP6]], splat (i64 1)
; CHECK-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i64> [[TMP7]], splat (i64 1)
; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[TMP8]], splat (i64 1)
; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP9]], splat (i64 1)
; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
; CHECK-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP11]], i64 4)
; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP30]], <vscale x 4 x i64> [[TMP12]], i64 8)
; CHECK-NEXT: [[TMP15:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP14]], <vscale x 4 x i64> [[TMP13]], i64 12)
; CHECK-NEXT: store <vscale x 16 x i64> [[TMP15]], ptr [[TMP3]], align 8, !alias.scope [[META3]], !noalias [[META6]]
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP17:%.*]] = load <vscale x 16 x i32>, ptr [[TMP18]], align 4, !alias.scope [[META6]]
; CHECK-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 0)
; CHECK-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 4)
; CHECK-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 8)
; CHECK-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 12)
; CHECK-NEXT: [[TMP23:%.*]] = add <vscale x 4 x i32> [[TMP19]], splat (i32 1)
; CHECK-NEXT: [[TMP24:%.*]] = add <vscale x 4 x i32> [[TMP20]], splat (i32 1)
; CHECK-NEXT: [[TMP25:%.*]] = add <vscale x 4 x i32> [[TMP21]], splat (i32 1)
; CHECK-NEXT: [[TMP26:%.*]] = add <vscale x 4 x i32> [[TMP22]], splat (i32 1)
; CHECK-NEXT: [[TMP32:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP23]], i64 0)
; CHECK-NEXT: [[TMP27:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP32]], <vscale x 4 x i32> [[TMP24]], i64 4)
; CHECK-NEXT: [[TMP28:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP27]], <vscale x 4 x i32> [[TMP25]], i64 8)
; CHECK-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP28]], <vscale x 4 x i32> [[TMP26]], i64 12)
; CHECK-NEXT: store <vscale x 16 x i32> [[TMP29]], ptr [[TMP18]], align 4, !alias.scope [[META6]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
;
; CHECK-UF2-TF-LABEL: define void @mixed_i64_i32_accesses(
; CHECK-UF2-TF-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-UF2-TF-NEXT: [[ENTRY:.*:]]
; CHECK-UF2-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-UF2-TF: [[VECTOR_PH]]:
; CHECK-UF2-TF-NEXT: [[TMP15:%.*]] = shl i64 [[TMP0]], 3
; CHECK-UF2-TF-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[X]], i64 [[TMP15]]
; CHECK-UF2-TF-NEXT: [[TMP17:%.*]] = shl i64 [[TMP0]], 2
; CHECK-UF2-TF-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[Y]], i64 [[TMP17]]
; CHECK-UF2-TF-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[X]], [[TMP24]]
; CHECK-UF2-TF-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[Y]], [[TMP16]]
; CHECK-UF2-TF-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-UF2-TF-NEXT: br i1 [[FOUND_CONFLICT]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH1:.*]]
; CHECK-UF2-TF: [[VECTOR_PH1]]:
; CHECK-UF2-TF-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF2-TF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 3
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP0]])
; CHECK-UF2-TF-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[TMP0]], i64 [[TMP3]])
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2-TF: [[VECTOR_BODY]]:
; CHECK-UF2-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH1]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
; CHECK-UF2-TF-NEXT: [[TMP8:%.*]] = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64.p0(ptr align 8 [[TMP4]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i64> poison), !alias.scope [[META3:![0-9]+]], !noalias [[META6:![0-9]+]]
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[TMP8]], i64 0)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[TMP8]], i64 4)
; CHECK-UF2-TF-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[WIDE_MASKED_LOAD8]], splat (i64 1)
; CHECK-UF2-TF-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[WIDE_MASKED_LOAD9]], splat (i64 1)
; CHECK-UF2-TF-NEXT: [[TMP18:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP12]], i64 0)
; CHECK-UF2-TF-NEXT: [[TMP19:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP18]], <vscale x 4 x i64> [[TMP13]], i64 4)
; CHECK-UF2-TF-NEXT: call void @llvm.masked.store.nxv8i64.p0(<vscale x 8 x i64> [[TMP19]], ptr align 8 [[TMP4]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3]], !noalias [[META6]]
; CHECK-UF2-TF-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
; CHECK-UF2-TF-NEXT: [[TMP25:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 4 [[TMP14]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> poison), !alias.scope [[META6]]
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[TMP25]], i64 0)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD13:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[TMP25]], i64 4)
; CHECK-UF2-TF-NEXT: [[TMP20:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD12]], splat (i32 1)
; CHECK-UF2-TF-NEXT: [[TMP21:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD13]], splat (i32 1)
; CHECK-UF2-TF-NEXT: [[TMP26:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP20]], i64 0)
; CHECK-UF2-TF-NEXT: [[TMP27:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP26]], <vscale x 4 x i32> [[TMP21]], i64 4)
; CHECK-UF2-TF-NEXT: call void @llvm.masked.store.nxv8i32.p0(<vscale x 8 x i32> [[TMP27]], ptr align 4 [[TMP14]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META6]]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TRIP_COUNT_MINUS_VF]])
; CHECK-UF2-TF-NEXT: [[TMP22:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-UF2-TF-NEXT: [[TMP23:%.*]] = xor i1 [[TMP22]], true
; CHECK-UF2-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-UF2-TF-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK-UF2-TF: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %next, %loop ]
%px = getelementptr inbounds i64, ptr %x, i64 %iv
%vx = load i64, ptr %px, align 8
%ax = add i64 %vx, 1
store i64 %ax, ptr %px, align 8
%py = getelementptr inbounds i32, ptr %y, i64 %iv
%vy = load i32, ptr %py, align 4
%ay = add i32 %vy, 1
store i32 %ay, ptr %py, align 4
%next = add nuw i64 %iv, 1
%cmp = icmp ult i64 %next, %n
br i1 %cmp, label %loop, label %exit
exit:
ret void
}
define void @mixed_i32_more_frequent_than_i64(ptr noalias %x, ptr noalias %y, ptr noalias %z, i64 %n) {
; CHECK-LABEL: define void @mixed_i32_more_frequent_than_i64(
; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], ptr noalias [[Z:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ITER_CHECK:.*:]]
; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP4:%.*]] = load <vscale x 16 x i64>, ptr [[TMP3]], align 8
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 0)
; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 4)
; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 8)
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv16i64(<vscale x 16 x i64> [[TMP4]], i64 12)
; CHECK-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i64> [[TMP6]], splat (i64 1)
; CHECK-NEXT: [[TMP11:%.*]] = add <vscale x 4 x i64> [[TMP7]], splat (i64 1)
; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[TMP8]], splat (i64 1)
; CHECK-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[TMP9]], splat (i64 1)
; CHECK-NEXT: [[TMP16:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> poison, <vscale x 4 x i64> [[TMP10]], i64 0)
; CHECK-NEXT: [[TMP30:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP16]], <vscale x 4 x i64> [[TMP11]], i64 4)
; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP30]], <vscale x 4 x i64> [[TMP12]], i64 8)
; CHECK-NEXT: [[TMP15:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP14]], <vscale x 4 x i64> [[TMP13]], i64 12)
; CHECK-NEXT: store <vscale x 16 x i64> [[TMP15]], ptr [[TMP3]], align 8
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP17:%.*]] = load <vscale x 16 x i32>, ptr [[TMP18]], align 4
; CHECK-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 0)
; CHECK-NEXT: [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 4)
; CHECK-NEXT: [[TMP21:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 8)
; CHECK-NEXT: [[TMP22:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP17]], i64 12)
; CHECK-NEXT: [[TMP23:%.*]] = add <vscale x 4 x i32> [[TMP19]], splat (i32 1)
; CHECK-NEXT: [[TMP24:%.*]] = add <vscale x 4 x i32> [[TMP20]], splat (i32 1)
; CHECK-NEXT: [[TMP25:%.*]] = add <vscale x 4 x i32> [[TMP21]], splat (i32 1)
; CHECK-NEXT: [[TMP26:%.*]] = add <vscale x 4 x i32> [[TMP22]], splat (i32 1)
; CHECK-NEXT: [[TMP45:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP23]], i64 0)
; CHECK-NEXT: [[TMP27:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP45]], <vscale x 4 x i32> [[TMP24]], i64 4)
; CHECK-NEXT: [[TMP28:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP27]], <vscale x 4 x i32> [[TMP25]], i64 8)
; CHECK-NEXT: [[TMP29:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP28]], <vscale x 4 x i32> [[TMP26]], i64 12)
; CHECK-NEXT: store <vscale x 16 x i32> [[TMP29]], ptr [[TMP18]], align 4
; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP46:%.*]] = load <vscale x 16 x i32>, ptr [[TMP31]], align 4
; CHECK-NEXT: [[TMP32:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 0)
; CHECK-NEXT: [[TMP33:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 4)
; CHECK-NEXT: [[TMP34:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 8)
; CHECK-NEXT: [[TMP35:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP46]], i64 12)
; CHECK-NEXT: [[TMP36:%.*]] = add <vscale x 4 x i32> [[TMP32]], splat (i32 2)
; CHECK-NEXT: [[TMP37:%.*]] = add <vscale x 4 x i32> [[TMP33]], splat (i32 2)
; CHECK-NEXT: [[TMP38:%.*]] = add <vscale x 4 x i32> [[TMP34]], splat (i32 2)
; CHECK-NEXT: [[TMP39:%.*]] = add <vscale x 4 x i32> [[TMP35]], splat (i32 2)
; CHECK-NEXT: [[TMP40:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP36]], i64 0)
; CHECK-NEXT: [[TMP41:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP40]], <vscale x 4 x i32> [[TMP37]], i64 4)
; CHECK-NEXT: [[TMP42:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP41]], <vscale x 4 x i32> [[TMP38]], i64 8)
; CHECK-NEXT: [[TMP43:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP42]], <vscale x 4 x i32> [[TMP39]], i64 12)
; CHECK-NEXT: store <vscale x 16 x i32> [[TMP43]], ptr [[TMP31]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
;
; CHECK-UF2-TF-LABEL: define void @mixed_i32_more_frequent_than_i64(
; CHECK-UF2-TF-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], ptr noalias [[Z:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-UF2-TF-NEXT: [[ENTRY:.*:]]
; CHECK-UF2-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-UF2-TF: [[VECTOR_PH]]:
; CHECK-UF2-TF-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF2-TF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 3
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP0]])
; CHECK-UF2-TF-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[TMP0]], i64 [[TMP3]])
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2-TF: [[VECTOR_BODY]]:
; CHECK-UF2-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[X]], i64 [[INDEX]]
; CHECK-UF2-TF-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64.p0(ptr align 8 [[TMP4]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i64> poison)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[TMP5]], i64 0)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 4 x i64> @llvm.vector.extract.nxv4i64.nxv8i64(<vscale x 8 x i64> [[TMP5]], i64 4)
; CHECK-UF2-TF-NEXT: [[TMP12:%.*]] = add <vscale x 4 x i64> [[WIDE_MASKED_LOAD8]], splat (i64 1)
; CHECK-UF2-TF-NEXT: [[TMP13:%.*]] = add <vscale x 4 x i64> [[WIDE_MASKED_LOAD9]], splat (i64 1)
; CHECK-UF2-TF-NEXT: [[TMP9:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> poison, <vscale x 4 x i64> [[TMP12]], i64 0)
; CHECK-UF2-TF-NEXT: [[TMP10:%.*]] = call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv4i64(<vscale x 8 x i64> [[TMP9]], <vscale x 4 x i64> [[TMP13]], i64 4)
; CHECK-UF2-TF-NEXT: call void @llvm.masked.store.nxv8i64.p0(<vscale x 8 x i64> [[TMP10]], ptr align 8 [[TMP4]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-UF2-TF-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[INDEX]]
; CHECK-UF2-TF-NEXT: [[TMP15:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 4 [[TMP14]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> poison)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[TMP15]], i64 0)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD13:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[TMP15]], i64 4)
; CHECK-UF2-TF-NEXT: [[TMP20:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD12]], splat (i32 1)
; CHECK-UF2-TF-NEXT: [[TMP21:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD13]], splat (i32 1)
; CHECK-UF2-TF-NEXT: [[TMP17:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP20]], i64 0)
; CHECK-UF2-TF-NEXT: [[TMP18:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP17]], <vscale x 4 x i32> [[TMP21]], i64 4)
; CHECK-UF2-TF-NEXT: call void @llvm.masked.store.nxv8i32.p0(<vscale x 8 x i32> [[TMP18]], ptr align 4 [[TMP14]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-UF2-TF-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 [[INDEX]]
; CHECK-UF2-TF-NEXT: [[TMP23:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 4 [[TMP22]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i32> poison)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD16:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[TMP23]], i64 0)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD17:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[TMP23]], i64 4)
; CHECK-UF2-TF-NEXT: [[TMP28:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD16]], splat (i32 2)
; CHECK-UF2-TF-NEXT: [[TMP29:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_LOAD17]], splat (i32 2)
; CHECK-UF2-TF-NEXT: [[TMP25:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP28]], i64 0)
; CHECK-UF2-TF-NEXT: [[TMP26:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP25]], <vscale x 4 x i32> [[TMP29]], i64 4)
; CHECK-UF2-TF-NEXT: call void @llvm.masked.store.nxv8i32.p0(<vscale x 8 x i32> [[TMP26]], ptr align 4 [[TMP22]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TRIP_COUNT_MINUS_VF]])
; CHECK-UF2-TF-NEXT: [[TMP30:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-UF2-TF-NEXT: [[TMP31:%.*]] = xor i1 [[TMP30]], true
; CHECK-UF2-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-UF2-TF-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK-UF2-TF: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %next, %loop ]
%px = getelementptr inbounds i64, ptr %x, i64 %iv
%vx = load i64, ptr %px, align 8
%ax = add i64 %vx, 1
store i64 %ax, ptr %px, align 8
%py = getelementptr inbounds i32, ptr %y, i64 %iv
%vy = load i32, ptr %py, align 4
%ay = add i32 %vy, 1
store i32 %ay, ptr %py, align 4
%pz = getelementptr inbounds i32, ptr %z, i64 %iv
%vz = load i32, ptr %pz, align 4
%az = add i32 %vz, 2
store i32 %az, ptr %pz, align 4
%next = add nuw i64 %iv, 1
%cmp = icmp ult i64 %next, %n
br i1 %cmp, label %loop, label %exit
exit:
ret void
}
define void @first_order_recurrence_i64_scaled_load_and_store(ptr noalias %src, ptr noalias %dst) {
; CHECK-LABEL: define void @first_order_recurrence_i64_scaled_load_and_store(
; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[FIRST:%.*]] = load i64, ptr [[SRC]], align 8
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <2 x i64> poison, i64 [[FIRST]], i32 1
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i64> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD3:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add nuw i64 1, [[INDEX]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 32
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 48
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8
; CHECK-NEXT: [[WIDE_LOAD3]] = load <2 x i64>, ptr [[TMP4]], align 8
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i64> [[VECTOR_RECUR]], <2 x i64> [[WIDE_LOAD]], <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD]], <2 x i64> [[WIDE_LOAD1]], <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD1]], <2 x i64> [[WIDE_LOAD2]], <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD2]], <2 x i64> [[WIDE_LOAD3]], <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: [[TMP9:%.*]] = add <2 x i64> [[TMP5]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP10:%.*]] = add <2 x i64> [[TMP6]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP11:%.*]] = add <2 x i64> [[TMP7]], [[WIDE_LOAD2]]
; CHECK-NEXT: [[TMP12:%.*]] = add <2 x i64> [[TMP8]], [[WIDE_LOAD3]]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 16
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 32
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP13]], i64 48
; CHECK-NEXT: store <2 x i64> [[TMP9]], ptr [[TMP13]], align 8
; CHECK-NEXT: store <2 x i64> [[TMP10]], ptr [[TMP14]], align 8
; CHECK-NEXT: store <2 x i64> [[TMP11]], ptr [[TMP15]], align 8
; CHECK-NEXT: store <2 x i64> [[TMP12]], ptr [[TMP16]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
;
; CHECK-UF2-TF-LABEL: define void @first_order_recurrence_i64_scaled_load_and_store(
; CHECK-UF2-TF-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
; CHECK-UF2-TF-NEXT: [[ENTRY:.*:]]
; CHECK-UF2-TF-NEXT: [[FIRST:%.*]] = load i64, ptr [[SRC]], align 8
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-UF2-TF: [[VECTOR_PH]]:
; CHECK-UF2-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF2-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1024)
; CHECK-UF2-TF-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i64 @llvm.usub.sat.i64(i64 1024, i64 [[TMP2]])
; CHECK-UF2-TF-NEXT: [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
; CHECK-UF2-TF-NEXT: [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 2
; CHECK-UF2-TF-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 1
; CHECK-UF2-TF-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[FIRST]], i32 [[TMP5]]
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2-TF: [[VECTOR_BODY]]:
; CHECK-UF2-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_MASKED_LOAD9:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[TMP6:%.*]] = add nuw i64 1, [[INDEX]]
; CHECK-UF2-TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP6]]
; CHECK-UF2-TF-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr align 8 [[TMP7]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i64> poison)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[TMP8]], i64 0)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD9]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[TMP8]], i64 2)
; CHECK-UF2-TF-NEXT: [[TMP15:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[WIDE_MASKED_LOAD7]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD8]], i32 1)
; CHECK-UF2-TF-NEXT: [[TMP16:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.right.nxv2i64(<vscale x 2 x i64> [[WIDE_MASKED_LOAD8]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD9]], i32 1)
; CHECK-UF2-TF-NEXT: [[TMP19:%.*]] = add <vscale x 2 x i64> [[TMP15]], [[WIDE_MASKED_LOAD8]]
; CHECK-UF2-TF-NEXT: [[TMP20:%.*]] = add <vscale x 2 x i64> [[TMP16]], [[WIDE_MASKED_LOAD9]]
; CHECK-UF2-TF-NEXT: [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP6]]
; CHECK-UF2-TF-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP19]], i64 0)
; CHECK-UF2-TF-NEXT: [[TMP18:%.*]] = call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP17]], <vscale x 2 x i64> [[TMP20]], i64 2)
; CHECK-UF2-TF-NEXT: call void @llvm.masked.store.nxv4i64.p0(<vscale x 4 x i64> [[TMP18]], ptr align 8 [[TMP21]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TRIP_COUNT_MINUS_VF]])
; CHECK-UF2-TF-NEXT: [[TMP25:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-UF2-TF-NEXT: [[TMP26:%.*]] = xor i1 [[TMP25]], true
; CHECK-UF2-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-UF2-TF-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK-UF2-TF: [[MIDDLE_BLOCK]]:
;
entry:
%first = load i64, ptr %src, align 8
br label %loop
loop:
%iv = phi i64 [ 1, %entry ], [ %next, %loop ]
%prev = phi i64 [ %first, %entry ], [ %cur, %loop ]
%cur.ptr = getelementptr inbounds i64, ptr %src, i64 %iv
%cur = load i64, ptr %cur.ptr, align 8
%sum = add i64 %prev, %cur
%dst.ptr = getelementptr inbounds i64, ptr %dst, i64 %iv
store i64 %sum, ptr %dst.ptr, align 8
%next = add nuw i64 %iv, 1
%cmp = icmp eq i64 %next, 1025
br i1 %cmp, label %exit, label %loop
exit:
ret void
}
define i64 @i64_sum_reduction_scaled_partial_reduce(ptr noalias %a, i64 %n) {
; CHECK-LABEL: define i64 @i64_sum_reduction_scaled_partial_reduce(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP13:%.*]] = load <vscale x 8 x i64>, ptr [[TMP3]], align 8
; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 0)
; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 2)
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 4)
; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP13]], i64 6)
; CHECK-NEXT: [[TMP8]] = add <vscale x 2 x i64> [[VEC_PHI]], [[TMP4]]
; CHECK-NEXT: [[TMP9]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP5]]
; CHECK-NEXT: [[TMP10]] = add <vscale x 2 x i64> [[VEC_PHI2]], [[TMP6]]
; CHECK-NEXT: [[TMP11]] = add <vscale x 2 x i64> [[VEC_PHI3]], [[TMP7]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
;
; CHECK-UF2-TF-LABEL: define i64 @i64_sum_reduction_scaled_partial_reduce(
; CHECK-UF2-TF-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-UF2-TF-NEXT: [[ENTRY:.*:]]
; CHECK-UF2-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-UF2-TF: [[VECTOR_PH]]:
; CHECK-UF2-TF-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF2-TF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
; CHECK-UF2-TF-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[TMP0]], i64 [[TMP3]])
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2-TF: [[VECTOR_BODY]]:
; CHECK-UF2-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[VEC_PHI8:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[VEC_PHI9:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK]], i64 0)
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK]], i64 2)
; CHECK-UF2-TF-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
; CHECK-UF2-TF-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr align 8 [[TMP4]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i64> poison)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[TMP6]], i64 0)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[TMP6]], i64 2)
; CHECK-UF2-TF-NEXT: [[TMP12:%.*]] = add <vscale x 2 x i64> [[VEC_PHI8]], [[WIDE_MASKED_LOAD11]]
; CHECK-UF2-TF-NEXT: [[TMP13:%.*]] = add <vscale x 2 x i64> [[VEC_PHI9]], [[WIDE_MASKED_LOAD12]]
; CHECK-UF2-TF-NEXT: [[TMP16]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]], <vscale x 2 x i64> [[TMP12]], <vscale x 2 x i64> [[VEC_PHI8]]
; CHECK-UF2-TF-NEXT: [[TMP17]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 2 x i64> [[TMP13]], <vscale x 2 x i64> [[VEC_PHI9]]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TRIP_COUNT_MINUS_VF]])
; CHECK-UF2-TF-NEXT: [[TMP18:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-UF2-TF-NEXT: [[TMP19:%.*]] = xor i1 [[TMP18]], true
; CHECK-UF2-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-UF2-TF-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK-UF2-TF: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %next, %loop ]
%acc = phi i64 [ 0, %entry ], [ %sum, %loop ]
%p = getelementptr inbounds i64, ptr %a, i64 %iv
%ld = load i64, ptr %p, align 8
%sum = add i64 %acc, %ld
%next = add nuw i64 %iv, 1
%cmp = icmp ult i64 %next, %n
br i1 %cmp, label %loop, label %exit
exit:
ret i64 %sum
}
define i64 @find_last_i64_scaled_load(i64 %n, ptr noalias %data, i64 %threshold) {
; CHECK-LABEL: define i64 @find_last_i64_scaled_load(
; CHECK-SAME: i64 [[N:%.*]], ptr noalias [[DATA:%.*]], i64 [[THRESHOLD:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[THRESHOLD]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP4:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP5:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP33:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP17:%.*]] = load <vscale x 8 x i64>, ptr [[TMP6]], align 8
; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 0)
; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 2)
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 4)
; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[TMP17]], i64 6)
; CHECK-NEXT: [[TMP11:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP8]]
; CHECK-NEXT: [[TMP13:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP9]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[TMP10]]
; CHECK-NEXT: [[TMP15:%.*]] = freeze <vscale x 2 x i1> [[TMP11]]
; CHECK-NEXT: [[TMP16:%.*]] = freeze <vscale x 2 x i1> [[TMP12]]
; CHECK-NEXT: [[TMP18:%.*]] = freeze <vscale x 2 x i1> [[TMP13]]
; CHECK-NEXT: [[TMP20:%.*]] = freeze <vscale x 2 x i1> [[TMP14]]
; CHECK-NEXT: [[TMP32:%.*]] = or <vscale x 2 x i1> [[TMP15]], [[TMP16]]
; CHECK-NEXT: [[TMP19:%.*]] = or <vscale x 2 x i1> [[TMP32]], [[TMP18]]
; CHECK-NEXT: [[TMP21:%.*]] = or <vscale x 2 x i1> [[TMP19]], [[TMP20]]
; CHECK-NEXT: [[TMP22:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP21]])
; CHECK-NEXT: [[TMP24]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP15]], <vscale x 2 x i1> [[TMP2]]
; CHECK-NEXT: [[TMP25]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP16]], <vscale x 2 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP26]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP18]], <vscale x 2 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP33]] = select i1 [[TMP22]], <vscale x 2 x i1> [[TMP20]], <vscale x 2 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP27]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP7]], <vscale x 2 x i64> [[VEC_PHI]]
; CHECK-NEXT: [[TMP28]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP8]], <vscale x 2 x i64> [[VEC_PHI1]]
; CHECK-NEXT: [[TMP29]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[VEC_PHI2]]
; CHECK-NEXT: [[TMP30]] = select i1 [[TMP22]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[VEC_PHI3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
;
; CHECK-UF2-TF-LABEL: define i64 @find_last_i64_scaled_load(
; CHECK-UF2-TF-SAME: i64 [[N:%.*]], ptr noalias [[DATA:%.*]], i64 [[THRESHOLD:%.*]]) #[[ATTR0]] {
; CHECK-UF2-TF-NEXT: [[ENTRY:.*:]]
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-UF2-TF: [[VECTOR_PH]]:
; CHECK-UF2-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF2-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
; CHECK-UF2-TF-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP2]])
; CHECK-UF2-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[THRESHOLD]], i64 0
; CHECK-UF2-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2-TF: [[VECTOR_BODY]]:
; CHECK-UF2-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[VEC_PHI8:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[VEC_PHI9:%.*]] = phi <vscale x 2 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP36:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[TMP5:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP31:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[TMP6:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK]], i64 0)
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK]], i64 2)
; CHECK-UF2-TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[INDEX]]
; CHECK-UF2-TF-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr align 8 [[TMP7]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i64> poison)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[TMP8]], i64 0)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[TMP8]], i64 2)
; CHECK-UF2-TF-NEXT: [[TMP15:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[WIDE_MASKED_LOAD11]]
; CHECK-UF2-TF-NEXT: [[TMP16:%.*]] = icmp slt <vscale x 2 x i64> [[BROADCAST_SPLAT]], [[WIDE_MASKED_LOAD12]]
; CHECK-UF2-TF-NEXT: [[TMP19:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK5]], <vscale x 2 x i1> [[TMP15]], <vscale x 2 x i1> zeroinitializer
; CHECK-UF2-TF-NEXT: [[TMP20:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 2 x i1> [[TMP16]], <vscale x 2 x i1> zeroinitializer
; CHECK-UF2-TF-NEXT: [[TMP23:%.*]] = freeze <vscale x 2 x i1> [[TMP19]]
; CHECK-UF2-TF-NEXT: [[TMP24:%.*]] = freeze <vscale x 2 x i1> [[TMP20]]
; CHECK-UF2-TF-NEXT: [[TMP27:%.*]] = or <vscale x 2 x i1> [[TMP23]], [[TMP24]]
; CHECK-UF2-TF-NEXT: [[TMP28:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[TMP27]])
; CHECK-UF2-TF-NEXT: [[TMP31]] = select i1 [[TMP28]], <vscale x 2 x i1> [[TMP23]], <vscale x 2 x i1> [[TMP5]]
; CHECK-UF2-TF-NEXT: [[TMP32]] = select i1 [[TMP28]], <vscale x 2 x i1> [[TMP24]], <vscale x 2 x i1> [[TMP6]]
; CHECK-UF2-TF-NEXT: [[TMP35]] = select i1 [[TMP28]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD11]], <vscale x 2 x i64> [[VEC_PHI8]]
; CHECK-UF2-TF-NEXT: [[TMP36]] = select i1 [[TMP28]], <vscale x 2 x i64> [[WIDE_MASKED_LOAD12]], <vscale x 2 x i64> [[VEC_PHI9]]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TRIP_COUNT_MINUS_VF]])
; CHECK-UF2-TF-NEXT: [[TMP37:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-UF2-TF-NEXT: [[TMP38:%.*]] = xor i1 [[TMP37]], true
; CHECK-UF2-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-UF2-TF-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK-UF2-TF: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %next, %loop ]
%data.phi = phi i64 [ -1, %entry ], [ %select.data, %loop ]
%p = getelementptr inbounds i64, ptr %data, i64 %iv
%ld = load i64, ptr %p, align 8
%select.cmp = icmp slt i64 %threshold, %ld
%select.data = select i1 %select.cmp, i64 %ld, i64 %data.phi
%next = add nuw i64 %iv, 1
%exit.cmp = icmp eq i64 %next, %n
br i1 %exit.cmp, label %exit, label %loop
exit:
ret i64 %select.data
}
; Negative test: We should not use a wide load when the result of the load is extended.
; In this case, it's better to use SVE extending loads (rather than a multi-vector load).
define void @extending_load(ptr noalias %dst, ptr %src, i64 %n) {
; CHECK-LABEL: define void @extending_load(
; CHECK-SAME: ptr noalias [[DST:%.*]], ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ITER_CHECK:.*:]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP2]], 1
; CHECK-NEXT: [[TMP5:%.*]] = mul nuw nsw i64 [[TMP2]], 3
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP3]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP3]], i64 [[TMP4]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP3]], i64 [[TMP5]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP3]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
; CHECK-NEXT: [[TMP9:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
; CHECK-NEXT: [[TMP10:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD2]] to <vscale x 4 x i64>
; CHECK-NEXT: [[TMP11:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD3]] to <vscale x 4 x i64>
; CHECK-NEXT: [[TMP12:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD4]] to <vscale x 4 x i64>
; CHECK-NEXT: [[TMP13:%.*]] = mul nsw <vscale x 4 x i64> [[TMP9]], splat (i64 42)
; CHECK-NEXT: [[TMP14:%.*]] = mul nsw <vscale x 4 x i64> [[TMP10]], splat (i64 42)
; CHECK-NEXT: [[TMP15:%.*]] = mul nsw <vscale x 4 x i64> [[TMP11]], splat (i64 42)
; CHECK-NEXT: [[TMP16:%.*]] = mul nsw <vscale x 4 x i64> [[TMP12]], splat (i64 42)
; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw i64, ptr [[DST]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP18:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> poison, <vscale x 4 x i64> [[TMP13]], i64 0)
; CHECK-NEXT: [[TMP19:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP18]], <vscale x 4 x i64> [[TMP14]], i64 4)
; CHECK-NEXT: [[TMP20:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP19]], <vscale x 4 x i64> [[TMP15]], i64 8)
; CHECK-NEXT: [[TMP21:%.*]] = call <vscale x 16 x i64> @llvm.vector.insert.nxv16i64.nxv4i64(<vscale x 16 x i64> [[TMP20]], <vscale x 4 x i64> [[TMP16]], i64 12)
; CHECK-NEXT: store <vscale x 16 x i64> [[TMP21]], ptr [[TMP17]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
;
; CHECK-UF2-TF-LABEL: define void @extending_load(
; CHECK-UF2-TF-SAME: ptr noalias [[DST:%.*]], ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-UF2-TF-NEXT: [[ENTRY:.*:]]
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-UF2-TF: [[VECTOR_PH]]:
; CHECK-UF2-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF2-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
; CHECK-UF2-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
; CHECK-UF2-TF-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP2]])
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2-TF: [[VECTOR_BODY]]:
; CHECK-UF2-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK1]], i64 0)
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = call <vscale x 2 x i1> @llvm.vector.extract.nxv2i1.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK1]], i64 2)
; CHECK-UF2-TF-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC]], i64 [[INDEX]]
; CHECK-UF2-TF-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP3]], i64 [[TMP1]]
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD8:%.*]] = call <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr align 4 [[TMP3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i32> poison)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr align 4 [[TMP8]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 2 x i32> poison)
; CHECK-UF2-TF-NEXT: [[TMP11:%.*]] = sext <vscale x 2 x i32> [[WIDE_MASKED_LOAD8]] to <vscale x 2 x i64>
; CHECK-UF2-TF-NEXT: [[TMP12:%.*]] = sext <vscale x 2 x i32> [[WIDE_MASKED_LOAD9]] to <vscale x 2 x i64>
; CHECK-UF2-TF-NEXT: [[TMP15:%.*]] = mul nsw <vscale x 2 x i64> [[TMP11]], splat (i64 42)
; CHECK-UF2-TF-NEXT: [[TMP16:%.*]] = mul nsw <vscale x 2 x i64> [[TMP12]], splat (i64 42)
; CHECK-UF2-TF-NEXT: [[TMP17:%.*]] = getelementptr inbounds nuw i64, ptr [[DST]], i64 [[INDEX]]
; CHECK-UF2-TF-NEXT: [[TMP14:%.*]] = call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP15]], i64 0)
; CHECK-UF2-TF-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP14]], <vscale x 2 x i64> [[TMP16]], i64 2)
; CHECK-UF2-TF-NEXT: call void @llvm.masked.store.nxv4i64.p0(<vscale x 4 x i64> [[TMP13]], ptr align 8 [[TMP17]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK1]])
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TRIP_COUNT_MINUS_VF]])
; CHECK-UF2-TF-NEXT: [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-UF2-TF-NEXT: [[TMP22:%.*]] = xor i1 [[TMP21]], true
; CHECK-UF2-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-UF2-TF-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK-UF2-TF: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%src.ptr = getelementptr inbounds nuw i32, ptr %src, i64 %iv
%src.val = load i32, ptr %src.ptr, align 4
%conv = sext i32 %src.val to i64
%mul = mul nsw i64 %conv, 42
%dst.ptr = getelementptr inbounds nuw i64, ptr %dst, i64 %iv
store i64 %mul, ptr %dst.ptr, align 8
%iv.next = add nuw nsw i64 %iv, 1
%exit.cmp = icmp eq i64 %iv.next, %n
br i1 %exit.cmp, label %exit, label %loop
exit:
ret void
}
; TODO: Support multi-vector operations with negative strides.
define void @reverse_stride(ptr noalias %c, ptr noalias %a, ptr noalias %b, i64 %n) {
; CHECK-LABEL: define void @reverse_stride(
; CHECK-SAME: ptr noalias [[C:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ITER_CHECK:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[N]], i64 0)
; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[SMIN]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP2]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP3]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[N]], [[N_VEC]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[INDEX]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = sub nuw nsw i64 [[TMP4]], 1
; CHECK-NEXT: [[TMP9:%.*]] = sub i64 0, [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i8], ptr [[TMP7]], i64 [[TMP9]]
; CHECK-NEXT: [[TMP27:%.*]] = sub i64 [[TMP9]], [[TMP4]]
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds [4 x i8], ptr [[TMP7]], i64 [[TMP27]]
; CHECK-NEXT: [[TMP29:%.*]] = mul i64 -2, [[TMP4]]
; CHECK-NEXT: [[TMP33:%.*]] = sub i64 [[TMP29]], [[TMP8]]
; CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds [4 x i8], ptr [[TMP7]], i64 [[TMP33]]
; CHECK-NEXT: [[TMP35:%.*]] = mul i64 -3, [[TMP4]]
; CHECK-NEXT: [[TMP36:%.*]] = sub i64 [[TMP35]], [[TMP8]]
; CHECK-NEXT: [[TMP37:%.*]] = getelementptr inbounds [4 x i8], ptr [[TMP7]], i64 [[TMP36]]
; CHECK-NEXT: [[TMP11:%.*]] = load <vscale x 4 x i32>, ptr [[TMP10]], align 4
; CHECK-NEXT: [[TMP12:%.*]] = load <vscale x 4 x i32>, ptr [[TMP28]], align 4
; CHECK-NEXT: [[TMP13:%.*]] = load <vscale x 4 x i32>, ptr [[TMP34]], align 4
; CHECK-NEXT: [[TMP14:%.*]] = load <vscale x 4 x i32>, ptr [[TMP37]], align 4
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[TMP6]]
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [4 x i8], ptr [[TMP15]], i64 [[TMP9]]
; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds [4 x i8], ptr [[TMP15]], i64 [[TMP27]]
; CHECK-NEXT: [[TMP39:%.*]] = getelementptr inbounds [4 x i8], ptr [[TMP15]], i64 [[TMP33]]
; CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds [4 x i8], ptr [[TMP15]], i64 [[TMP36]]
; CHECK-NEXT: [[TMP17:%.*]] = load <vscale x 4 x i32>, ptr [[TMP16]], align 4
; CHECK-NEXT: [[TMP18:%.*]] = load <vscale x 4 x i32>, ptr [[TMP38]], align 4
; CHECK-NEXT: [[TMP19:%.*]] = load <vscale x 4 x i32>, ptr [[TMP39]], align 4
; CHECK-NEXT: [[TMP20:%.*]] = load <vscale x 4 x i32>, ptr [[TMP40]], align 4
; CHECK-NEXT: [[TMP21:%.*]] = add nsw <vscale x 4 x i32> [[TMP17]], [[TMP11]]
; CHECK-NEXT: [[TMP22:%.*]] = add nsw <vscale x 4 x i32> [[TMP18]], [[TMP12]]
; CHECK-NEXT: [[TMP23:%.*]] = add nsw <vscale x 4 x i32> [[TMP19]], [[TMP13]]
; CHECK-NEXT: [[TMP24:%.*]] = add nsw <vscale x 4 x i32> [[TMP20]], [[TMP14]]
; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[TMP6]]
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds [4 x i8], ptr [[TMP25]], i64 [[TMP9]]
; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds [4 x i8], ptr [[TMP25]], i64 [[TMP27]]
; CHECK-NEXT: [[TMP41:%.*]] = getelementptr inbounds [4 x i8], ptr [[TMP25]], i64 [[TMP33]]
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds [4 x i8], ptr [[TMP25]], i64 [[TMP36]]
; CHECK-NEXT: store <vscale x 4 x i32> [[TMP21]], ptr [[TMP26]], align 4
; CHECK-NEXT: store <vscale x 4 x i32> [[TMP22]], ptr [[TMP30]], align 4
; CHECK-NEXT: store <vscale x 4 x i32> [[TMP23]], ptr [[TMP41]], align 4
; CHECK-NEXT: store <vscale x 4 x i32> [[TMP24]], ptr [[TMP32]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
;
; CHECK-UF2-TF-LABEL: define void @reverse_stride(
; CHECK-UF2-TF-SAME: ptr noalias [[C:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-UF2-TF-NEXT: [[ENTRY:.*:]]
; CHECK-UF2-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.smin.i64(i64 [[N]], i64 0)
; CHECK-UF2-TF-NEXT: [[TMP1:%.*]] = add i64 [[N]], 1
; CHECK-UF2-TF-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], [[TMP0]]
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-UF2-TF: [[VECTOR_PH]]:
; CHECK-UF2-TF-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF2-TF-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
; CHECK-UF2-TF-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP3]], 3
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP2]])
; CHECK-UF2-TF-NEXT: [[EXTRACT_ENTRY_ALM_PART:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 0)
; CHECK-UF2-TF-NEXT: [[EXTRACT_ENTRY_ALM_PART1:%.*]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], i64 4)
; CHECK-UF2-TF-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[TMP2]], i64 [[TMP5]])
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2-TF: [[VECTOR_BODY]]:
; CHECK-UF2-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK5:%.*]] = phi <vscale x 4 x i1> [ [[EXTRACT_ENTRY_ALM_PART]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 4 x i1> [ [[EXTRACT_ENTRY_ALM_PART1]], %[[VECTOR_PH]] ], [ [[EXTRACT_NEXT_ALM_PART7:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[INDEX]]
; CHECK-UF2-TF-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[TMP6]]
; CHECK-UF2-TF-NEXT: [[TMP8:%.*]] = sub nuw nsw i64 [[TMP4]], 1
; CHECK-UF2-TF-NEXT: [[TMP9:%.*]] = sub i64 0, [[TMP8]]
; CHECK-UF2-TF-NEXT: [[TMP15:%.*]] = getelementptr [4 x i8], ptr [[TMP7]], i64 [[TMP9]]
; CHECK-UF2-TF-NEXT: [[TMP17:%.*]] = sub i64 [[TMP9]], [[TMP4]]
; CHECK-UF2-TF-NEXT: [[TMP18:%.*]] = getelementptr [4 x i8], ptr [[TMP7]], i64 [[TMP17]]
; CHECK-UF2-TF-NEXT: [[REVERSE8:%.*]] = call <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK5]])
; CHECK-UF2-TF-NEXT: [[REVERSE9:%.*]] = call <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK6]])
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP15]], <vscale x 4 x i1> [[REVERSE8]], <vscale x 4 x i32> poison)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP18]], <vscale x 4 x i1> [[REVERSE9]], <vscale x 4 x i32> poison)
; CHECK-UF2-TF-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[TMP6]]
; CHECK-UF2-TF-NEXT: [[TMP22:%.*]] = getelementptr [4 x i8], ptr [[TMP19]], i64 [[TMP9]]
; CHECK-UF2-TF-NEXT: [[TMP23:%.*]] = getelementptr [4 x i8], ptr [[TMP19]], i64 [[TMP17]]
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD15:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP22]], <vscale x 4 x i1> [[REVERSE8]], <vscale x 4 x i32> poison)
; CHECK-UF2-TF-NEXT: [[WIDE_MASKED_LOAD16:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP23]], <vscale x 4 x i1> [[REVERSE9]], <vscale x 4 x i32> poison)
; CHECK-UF2-TF-NEXT: [[TMP26:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD15]], [[WIDE_MASKED_LOAD11]]
; CHECK-UF2-TF-NEXT: [[TMP27:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD16]], [[WIDE_MASKED_LOAD12]]
; CHECK-UF2-TF-NEXT: [[TMP28:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i64 [[TMP6]]
; CHECK-UF2-TF-NEXT: [[TMP31:%.*]] = getelementptr [4 x i8], ptr [[TMP28]], i64 [[TMP9]]
; CHECK-UF2-TF-NEXT: [[TMP32:%.*]] = getelementptr [4 x i8], ptr [[TMP28]], i64 [[TMP17]]
; CHECK-UF2-TF-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP26]], ptr align 4 [[TMP31]], <vscale x 4 x i1> [[REVERSE8]])
; CHECK-UF2-TF-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP27]], ptr align 4 [[TMP32]], <vscale x 4 x i1> [[REVERSE9]])
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TRIP_COUNT_MINUS_VF]])
; CHECK-UF2-TF-NEXT: [[EXTRACT_NEXT_ALM_PART]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0)
; CHECK-UF2-TF-NEXT: [[EXTRACT_NEXT_ALM_PART7]] = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1(<vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 4)
; CHECK-UF2-TF-NEXT: [[TMP33:%.*]] = extractelement <vscale x 4 x i1> [[EXTRACT_NEXT_ALM_PART]], i64 0
; CHECK-UF2-TF-NEXT: [[TMP34:%.*]] = xor i1 [[TMP33]], true
; CHECK-UF2-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]]
; CHECK-UF2-TF-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK-UF2-TF: [[MIDDLE_BLOCK]]:
;
entry:
br label %for.body
for.body:
%iv = phi i64 [ %dec, %for.body ], [ %n, %entry ]
%a.ptr = getelementptr inbounds nuw i32, ptr %a, i64 %iv
%a.val = load i32, ptr %a.ptr, align 4
%b.ptr = getelementptr inbounds nuw i32, ptr %b, i64 %iv
%b.val = load i32, ptr %b.ptr, align 4
%add = add nsw i32 %b.val, %a.val
%c.ptr = getelementptr inbounds nuw i32, ptr %c, i64 %iv
store i32 %add, ptr %c.ptr, align 4
%dec = add nsw i64 %iv, -1
%cmp = icmp sgt i64 %iv, 0
br i1 %cmp, label %for.body, label %exit
exit:
ret void
}
; Negative test (load): A strided load (stride = 2). Only the store should be widened to a VF multiple.
define void @gather_nxv4i32_stride2(ptr noalias %a, ptr noalias %b, i64 %n) {
; CHECK-LABEL: define void @gather_nxv4i32_stride2(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ITER_CHECK:.*:]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ule i64 [[N]], [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i64 [[TMP1]], i64 [[N_MOD_VF]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP4]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP2]], 1
; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP2]], 1
; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP8]], 1
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP2]], 3
; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP12]], 1
; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[TMP14]]
; CHECK-NEXT: [[TMP16:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP17:%.*]] = shl i64 [[TMP7]], 1
; CHECK-NEXT: [[TMP18:%.*]] = shl i64 [[TMP11]], 1
; CHECK-NEXT: [[TMP19:%.*]] = shl i64 [[TMP15]], 1
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP16]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP17]]
; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP18]]
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP19]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP20]], align 4
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
; CHECK-NEXT: [[TMP24:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC2]])
; CHECK-NEXT: [[TMP25:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC3]], 0
; CHECK-NEXT: [[WIDE_VEC4:%.*]] = load <vscale x 8 x float>, ptr [[TMP22]], align 4
; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC4]])
; CHECK-NEXT: [[TMP26:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC5]], 0
; CHECK-NEXT: [[WIDE_VEC6:%.*]] = load <vscale x 8 x float>, ptr [[TMP23]], align 4
; CHECK-NEXT: [[STRIDED_VEC7:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC6]])
; CHECK-NEXT: [[TMP27:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC7]], 0
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP30:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP24]], i64 0)
; CHECK-NEXT: [[TMP31:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP30]], <vscale x 4 x float> [[TMP25]], i64 4)
; CHECK-NEXT: [[TMP32:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP31]], <vscale x 4 x float> [[TMP26]], i64 8)
; CHECK-NEXT: [[TMP29:%.*]] = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP32]], <vscale x 4 x float> [[TMP27]], i64 12)
; CHECK-NEXT: store <vscale x 16 x float> [[TMP29]], ptr [[TMP28]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
;
; CHECK-UF2-TF-LABEL: define void @gather_nxv4i32_stride2(
; CHECK-UF2-TF-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-UF2-TF-NEXT: [[ENTRY:.*:]]
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-UF2-TF: [[VECTOR_PH]]:
; CHECK-UF2-TF-NEXT: [[TMP0:%.*]] = icmp ult i64 0, [[N]]
; CHECK-UF2-TF-NEXT: [[TMP1:%.*]] = icmp ult i64 1, [[N]]
; CHECK-UF2-TF-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 2)
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2-TF: [[VECTOR_BODY]]:
; CHECK-UF2-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE9:.*]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[TMP27:%.*]], %[[PRED_STORE_CONTINUE9]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK3:%.*]] = phi i1 [ [[TMP1]], %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[PRED_STORE_CONTINUE9]] ]
; CHECK-UF2-TF-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 1
; CHECK-UF2-TF-NEXT: [[TMP9:%.*]] = shl i64 [[INDEX]], 1
; CHECK-UF2-TF-NEXT: [[TMP10:%.*]] = shl i64 [[TMP4]], 1
; CHECK-UF2-TF-NEXT: br i1 [[ACTIVE_LANE_MASK2]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; CHECK-UF2-TF: [[PRED_STORE_IF6]]:
; CHECK-UF2-TF-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP9]]
; CHECK-UF2-TF-NEXT: [[TMP18:%.*]] = load float, ptr [[TMP17]], align 4
; CHECK-UF2-TF-NEXT: [[TMP19:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
; CHECK-UF2-TF-NEXT: store float [[TMP18]], ptr [[TMP19]], align 4
; CHECK-UF2-TF-NEXT: br label %[[PRED_STORE_CONTINUE7]]
; CHECK-UF2-TF: [[PRED_STORE_CONTINUE7]]:
; CHECK-UF2-TF-NEXT: br i1 [[ACTIVE_LANE_MASK3]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9]]
; CHECK-UF2-TF: [[PRED_STORE_IF8]]:
; CHECK-UF2-TF-NEXT: [[TMP20:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP10]]
; CHECK-UF2-TF-NEXT: [[TMP21:%.*]] = load float, ptr [[TMP20]], align 4
; CHECK-UF2-TF-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP4]]
; CHECK-UF2-TF-NEXT: store float [[TMP21]], ptr [[TMP22]], align 4
; CHECK-UF2-TF-NEXT: br label %[[PRED_STORE_CONTINUE9]]
; CHECK-UF2-TF: [[PRED_STORE_CONTINUE9]]:
; CHECK-UF2-TF-NEXT: [[TMP27]] = icmp ult i64 [[INDEX]], [[TRIP_COUNT_MINUS_VF]]
; CHECK-UF2-TF-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 1
; CHECK-UF2-TF-NEXT: [[TMP13]] = icmp ult i64 [[TMP12]], [[TRIP_COUNT_MINUS_VF]]
; CHECK-UF2-TF-NEXT: [[TMP14:%.*]] = xor i1 [[TMP27]], true
; CHECK-UF2-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2
; CHECK-UF2-TF-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK-UF2-TF: [[MIDDLE_BLOCK]]:
;
entry:
br label %for.body
for.body:
%iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
%iv.stride2 = mul i64 %iv, 2
%b.ptr = getelementptr inbounds float, ptr %b, i64 %iv.stride2
%b.val = load float, ptr %b.ptr, align 4
%a.ptr = getelementptr inbounds float, ptr %a, i64 %iv
store float %b.val, ptr %a.ptr, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, %n
br i1 %exitcond.not, label %exit, label %for.body
exit:
ret void
}
define void @store_address(ptr noalias %x, ptr noalias %y, i64 %n) {
; CHECK-LABEL: define void @store_address(
; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 1
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP2]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG31:![0-9]+]]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG31]]
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]], !dbg [[DBG31]]
; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <vscale x 2 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]], !dbg [[DBG31]]
; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <vscale x 2 x i64> [[STEP_ADD_2]], [[BROADCAST_SPLAT]], !dbg [[DBG31]]
; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds [8 x i8], ptr [[X]], <vscale x 2 x i64> [[VEC_IND]], !dbg [[DBG35:![0-9]+]]
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <vscale x 2 x ptr> [[WIDE_GEP]], i64 0
; CHECK-NEXT: [[WIDE_GEP1:%.*]] = getelementptr inbounds [8 x i8], ptr [[X]], <vscale x 2 x i64> [[STEP_ADD]], !dbg [[DBG35]]
; CHECK-NEXT: [[WIDE_GEP2:%.*]] = getelementptr inbounds [8 x i8], ptr [[X]], <vscale x 2 x i64> [[STEP_ADD_2]], !dbg [[DBG35]]
; CHECK-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds [8 x i8], ptr [[X]], <vscale x 2 x i64> [[STEP_ADD_3]], !dbg [[DBG35]]
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> poison, <vscale x 2 x ptr> [[WIDE_GEP]], i64 0), !dbg [[DBG36:![0-9]+]]
; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP6]], <vscale x 2 x ptr> [[WIDE_GEP1]], i64 2), !dbg [[DBG36]]
; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP7]], <vscale x 2 x ptr> [[WIDE_GEP2]], i64 4), !dbg [[DBG36]]
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 8 x ptr> @llvm.vector.insert.nxv8p0.nxv2p0(<vscale x 8 x ptr> [[TMP8]], <vscale x 2 x ptr> [[WIDE_GEP3]], i64 6), !dbg [[DBG36]]
; CHECK-NEXT: store <vscale x 8 x ptr> [[TMP9]], ptr [[TMP5]], align 8, !dbg [[DBG36]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]], !dbg [[DBG31]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT]], !dbg [[DBG31]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]], !dbg [[DBG37:![0-9]+]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !dbg [[DBG37]], !llvm.loop [[LOOP38:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
;
; CHECK-UF2-TF-LABEL: define void @store_address(
; CHECK-UF2-TF-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-UF2-TF-NEXT: [[ENTRY:.*:]]
; CHECK-UF2-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-UF2-TF: [[VECTOR_PH]]:
; CHECK-UF2-TF-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF2-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
; CHECK-UF2-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP2]], i64 0
; CHECK-UF2-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
; CHECK-UF2-TF-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 2
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]]), !dbg [[DBG19:![0-9]+]]
; CHECK-UF2-TF-NEXT: [[TRIP_COUNT_MINUS_VF:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[TMP0]], i64 [[TMP3]]), !dbg [[DBG19]]
; CHECK-UF2-TF-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2-TF: [[VECTOR_BODY]]:
; CHECK-UF2-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG19]]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG19]]
; CHECK-UF2-TF-NEXT: [[STEP_ADD:%.*]] = add nuw <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]], !dbg [[DBG19]]
; CHECK-UF2-TF-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds [8 x i8], ptr [[X]], <vscale x 2 x i64> [[VEC_IND]], !dbg [[DBG23:![0-9]+]]
; CHECK-UF2-TF-NEXT: [[TMP5:%.*]] = extractelement <vscale x 2 x ptr> [[WIDE_GEP]], i64 0
; CHECK-UF2-TF-NEXT: [[WIDE_GEP1:%.*]] = getelementptr inbounds [8 x i8], ptr [[X]], <vscale x 2 x i64> [[STEP_ADD]], !dbg [[DBG23]]
; CHECK-UF2-TF-NEXT: [[TMP6:%.*]] = call <vscale x 4 x ptr> @llvm.vector.insert.nxv4p0.nxv2p0(<vscale x 4 x ptr> poison, <vscale x 2 x ptr> [[WIDE_GEP]], i64 0), !dbg [[DBG24:![0-9]+]]
; CHECK-UF2-TF-NEXT: [[TMP7:%.*]] = call <vscale x 4 x ptr> @llvm.vector.insert.nxv4p0.nxv2p0(<vscale x 4 x ptr> [[TMP6]], <vscale x 2 x ptr> [[WIDE_GEP1]], i64 2), !dbg [[DBG24]]
; CHECK-UF2-TF-NEXT: call void @llvm.masked.store.nxv4p0.p0(<vscale x 4 x ptr> [[TMP7]], ptr align 8 [[TMP5]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]]), !dbg [[DBG24]]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TRIP_COUNT_MINUS_VF]]), !dbg [[DBG19]]
; CHECK-UF2-TF-NEXT: [[TMP8:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0, !dbg [[DBG19]]
; CHECK-UF2-TF-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true, !dbg [[DBG19]]
; CHECK-UF2-TF-NEXT: [[VEC_IND_NEXT]] = add nuw <vscale x 2 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]], !dbg [[DBG19]]
; CHECK-UF2-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]], !dbg [[DBG19]]
; CHECK-UF2-TF-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !dbg [[DBG19]], !llvm.loop [[LOOP25:![0-9]+]]
; CHECK-UF2-TF: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %next, %loop ], !dbg !3
%px = getelementptr inbounds i64, ptr %x, i64 %iv, !dbg !4
store ptr %px, ptr %px, align 8, !dbg !5
%next = add nuw i64 %iv, 1, !dbg !6
%cmp = icmp ult i64 %next, %n, !dbg !7
br i1 %cmp, label %loop, label %exit, !dbg !8
exit:
ret void
}
; Tests removing the backedge branch for a tail-folded multi-vector loop.
define void @i32_single_iteration(ptr noalias %x) {
; CHECK-LABEL: define void @i32_single_iteration(
; CHECK-SAME: ptr noalias [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ITER_CHECK:.*:]]
; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TRIP_COUNT:%.*]] = mul nuw i64 [[VSCALE]], 8
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TRIP_COUNT]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[VEC_EPILOG_SCALAR_PH:label %.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
; CHECK-NEXT: br i1 true, [[VEC_EPILOG_PH:label %.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TRIP_COUNT]], [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TRIP_COUNT]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP2:%.*]] = load <vscale x 16 x i32>, ptr [[X]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP2]], i64 0)
; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP2]], i64 4)
; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP2]], i64 8)
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[TMP2]], i64 12)
; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[TMP3]], splat (i32 1)
; CHECK-NEXT: [[TMP8:%.*]] = add <vscale x 4 x i32> [[TMP4]], splat (i32 1)
; CHECK-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP5]], splat (i32 1)
; CHECK-NEXT: [[TMP10:%.*]] = add <vscale x 4 x i32> [[TMP6]], splat (i32 1)
; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP7]], i64 0)
; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP11]], <vscale x 4 x i32> [[TMP8]], i64 4)
; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP12]], <vscale x 4 x i32> [[TMP9]], i64 8)
; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP13]], <vscale x 4 x i32> [[TMP10]], i64 12)
; CHECK-NEXT: store <vscale x 16 x i32> [[TMP14]], ptr [[X]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
;
; CHECK-UF2-TF-LABEL: define void @i32_single_iteration(
; CHECK-UF2-TF-SAME: ptr noalias [[X:%.*]]) #[[ATTR0]] {
; CHECK-UF2-TF-NEXT: [[ENTRY:.*:]]
; CHECK-UF2-TF-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF2-TF-NEXT: [[TRIP_COUNT:%.*]] = mul nuw i64 [[VSCALE]], 8
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-UF2-TF: [[VECTOR_PH]]:
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TRIP_COUNT]])
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2-TF: [[VECTOR_BODY]]:
; CHECK-UF2-TF-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr align 4 [[X]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]], <vscale x 8 x i32> poison)
; CHECK-UF2-TF-NEXT: [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[TMP0]], i64 0)
; CHECK-UF2-TF-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[TMP0]], i64 4)
; CHECK-UF2-TF-NEXT: [[TMP3:%.*]] = add <vscale x 4 x i32> [[TMP1]], splat (i32 1)
; CHECK-UF2-TF-NEXT: [[TMP4:%.*]] = add <vscale x 4 x i32> [[TMP2]], splat (i32 1)
; CHECK-UF2-TF-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP3]], i64 0)
; CHECK-UF2-TF-NEXT: [[TMP6:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP4]], i64 4)
; CHECK-UF2-TF-NEXT: call void @llvm.masked.store.nxv8i32.p0(<vscale x 8 x i32> [[TMP6]], ptr align 4 [[X]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK_ENTRY]])
; CHECK-UF2-TF-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK-UF2-TF: [[MIDDLE_BLOCK]]:
;
entry:
%vscale = call i64 @llvm.vscale.i64()
%trip.count = mul nuw i64 %vscale, 8
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %next, %loop ]
%x.ptr = getelementptr inbounds i32, ptr %x, i64 %iv
%x.val = load i32, ptr %x.ptr, align 4
%add = add i32 %x.val, 1
store i32 %add, ptr %x.ptr, align 4
%next = add nuw i64 %iv, 1
%cmp = icmp eq i64 %next, %trip.count
br i1 %cmp, label %exit, label %loop
exit:
ret void
}
; Tests constant folding the wide-active-lane-mask to all true.
define void @i32_fold_wide_active_lane_mask(ptr noalias %x) vscale_range(1, 2) {
; CHECK-LABEL: define void @i32_fold_wide_active_lane_mask(
; CHECK-SAME: ptr noalias [[X:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 16
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 32
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 48
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[X]], align 4
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 1)
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[WIDE_LOAD1]], splat (i32 1)
; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[WIDE_LOAD2]], splat (i32 1)
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[WIDE_LOAD3]], splat (i32 1)
; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[X]], align 4
; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP0]], align 4
; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP1]], align 4
; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP2]], align 4
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
;
; CHECK-UF2-TF-LABEL: define void @i32_fold_wide_active_lane_mask(
; CHECK-UF2-TF-SAME: ptr noalias [[X:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-UF2-TF-NEXT: [[ENTRY:.*:]]
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-UF2-TF: [[VECTOR_PH]]:
; CHECK-UF2-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF2-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2-TF: [[VECTOR_BODY]]:
; CHECK-UF2-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[INDEX]]
; CHECK-UF2-TF-NEXT: [[TMP3:%.*]] = load <vscale x 8 x i32>, ptr [[TMP2]], align 4
; CHECK-UF2-TF-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[TMP3]], i64 0)
; CHECK-UF2-TF-NEXT: [[TMP5:%.*]] = call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[TMP3]], i64 4)
; CHECK-UF2-TF-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i32> [[TMP4]], splat (i32 1)
; CHECK-UF2-TF-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[TMP5]], splat (i32 1)
; CHECK-UF2-TF-NEXT: [[TMP8:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP6]], i64 0)
; CHECK-UF2-TF-NEXT: [[TMP9:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP8]], <vscale x 4 x i32> [[TMP7]], i64 4)
; CHECK-UF2-TF-NEXT: store <vscale x 8 x i32> [[TMP9]], ptr [[TMP2]], align 4
; CHECK-UF2-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-UF2-TF-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
; CHECK-UF2-TF-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
; CHECK-UF2-TF: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %next, %loop ]
%x.ptr = getelementptr inbounds i32, ptr %x, i64 %iv
%x.val = load i32, ptr %x.ptr, align 4
%add = add i32 %x.val, 1
store i32 %add, ptr %x.ptr, align 4
%next = add nuw i64 %iv, 1
%cmp = icmp eq i64 %next, 16
br i1 %cmp, label %exit, label %loop
exit:
ret void
}
define void @wide_active_lane_mask_all_true_incoming(ptr noalias %p) vscale_range(1,2) {
; CHECK-LABEL: define void @wide_active_lane_mask_all_true_incoming(
; CHECK-SAME: ptr noalias [[P:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 17, [[TMP1]]
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP2]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 17, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 17, [[N_MOD_VF]]
; CHECK-NEXT: [[TMP3:%.*]] = trunc <vscale x 4 x i64> [[BROADCAST_SPLAT]] to <vscale x 4 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[STEP_ADD:%.*]] = add <vscale x 4 x i32> [[VEC_IND]], [[TMP3]]
; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add <vscale x 4 x i32> [[STEP_ADD]], [[TMP3]]
; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add <vscale x 4 x i32> [[STEP_ADD_2]], [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[VEC_IND]], i64 0)
; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[STEP_ADD]], i64 4)
; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[STEP_ADD_2]], i64 8)
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP8]], <vscale x 4 x i32> [[STEP_ADD_3]], i64 12)
; CHECK-NEXT: store <vscale x 16 x i32> [[TMP9]], ptr [[TMP5]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[STEP_ADD_3]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
;
; CHECK-UF2-TF-LABEL: define void @wide_active_lane_mask_all_true_incoming(
; CHECK-UF2-TF-SAME: ptr noalias [[P:%.*]]) #[[ATTR1]] {
; CHECK-UF2-TF-NEXT: [[ENTRY:.*:]]
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK-UF2-TF: [[VECTOR_PH]]:
; CHECK-UF2-TF-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-UF2-TF-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-UF2-TF-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP1]], i64 0
; CHECK-UF2-TF-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-UF2-TF-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 3
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 17)
; CHECK-UF2-TF-NEXT: [[TMP3:%.*]] = trunc <vscale x 4 x i64> [[BROADCAST_SPLAT]] to <vscale x 4 x i32>
; CHECK-UF2-TF-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
; CHECK-UF2-TF-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-UF2-TF: [[VECTOR_BODY]]:
; CHECK-UF2-TF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-UF2-TF-NEXT: [[STEP_ADD:%.*]] = add <vscale x 4 x i32> [[VEC_IND]], [[TMP3]]
; CHECK-UF2-TF-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
; CHECK-UF2-TF-NEXT: [[TMP6:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[VEC_IND]], i64 0)
; CHECK-UF2-TF-NEXT: [[TMP7:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP6]], <vscale x 4 x i32> [[STEP_ADD]], i64 4)
; CHECK-UF2-TF-NEXT: call void @llvm.masked.store.nxv8i32.p0(<vscale x 8 x i32> [[TMP7]], ptr align 4 [[TMP5]], <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-UF2-TF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; CHECK-UF2-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 17)
; CHECK-UF2-TF-NEXT: [[TMP8:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-UF2-TF-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true
; CHECK-UF2-TF-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[STEP_ADD]], [[TMP3]]
; CHECK-UF2-TF-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; CHECK-UF2-TF: [[MIDDLE_BLOCK]]:
;
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%t = trunc i64 %iv to i32
%gep = getelementptr i32, ptr %p, i64 %iv
store i32 %t, ptr %gep, align 4
%iv.next = add i64 %iv, 1
%done = icmp eq i64 %iv.next, 17
br i1 %done, label %exit, label %loop
exit:
ret void
}
!llvm.dbg.cu = !{!0}
!llvm.module.flags = !{!2}
!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 24.0.0git")
!1 = !DIFile(filename: "test.cpp", directory: "/")
!2 = !{i32 2, !"Debug Info Version", i32 3}
!3 = !DILocation(line: 4, scope: !9)
!4 = !DILocation(line: 5, scope: !9)
!5 = !DILocation(line: 6, scope: !9)
!6 = !DILocation(line: 7, scope: !9)
!7 = !DILocation(line: 8, scope: !9)
!8 = !DILocation(line: 9, scope: !9)
!9 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 11, type: !10, unit: !0, retainedNodes: !11)
!10 = distinct !DISubroutineType(types: !11)
!11 = !{}