| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 4 |
| ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVE1 |
| ; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVED |
| ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-MAXBW |
| |
| target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" |
| target triple = "aarch64-none-unknown-elf" |
| |
| define i32 @dotp(ptr %a, ptr %b) #0 { |
| ; CHECK-INTERLEAVE1-LABEL: define i32 @dotp( |
| ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; CHECK-INTERLEAVE1-NEXT: entry: |
| ; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_PH:%.*]] |
| ; CHECK-INTERLEAVE1: vector.ph: |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4 |
| ; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_BODY:%.*]] |
| ; CHECK-INTERLEAVE1: vector.body: |
| ; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]] |
| ; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP7]], align 1 |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]] |
| ; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP10]], align 1 |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32> |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32> |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = mul <vscale x 16 x i32> [[TMP4]], [[TMP5]] |
| ; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP6]]) |
| ; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]] |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 |
| ; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; CHECK-INTERLEAVE1: middle.block: |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]]) |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = sub i32 0, [[TMP9]] |
| ; CHECK-INTERLEAVE1-NEXT: br label [[FOR_EXIT:%.*]] |
| ; CHECK-INTERLEAVE1: for.exit: |
| ; CHECK-INTERLEAVE1-NEXT: ret i32 [[TMP11]] |
| ; |
| ; CHECK-INTERLEAVED-LABEL: define i32 @dotp( |
| ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; CHECK-INTERLEAVED-NEXT: entry: |
| ; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_PH:%.*]] |
| ; CHECK-INTERLEAVED: vector.ph: |
| ; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() |
| ; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP0]], 4 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 5 |
| ; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_BODY:%.*]] |
| ; CHECK-INTERLEAVED: vector.body: |
| ; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP7]], i64 [[TMP4]] |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP7]], align 1 |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP4]] |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i8>, ptr [[TMP14]], align 1 |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 16 x i8>, ptr [[TMP9]], align 1 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD3]] to <vscale x 16 x i32> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = mul <vscale x 16 x i32> [[TMP10]], [[TMP11]] |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP12]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD4]] to <vscale x 16 x i32> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = mul <vscale x 16 x i32> [[TMP21]], [[TMP15]] |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE5]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP16]]) |
| ; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 |
| ; CHECK-INTERLEAVED-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; CHECK-INTERLEAVED: middle.block: |
| ; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = sub i32 0, [[TMP19]] |
| ; CHECK-INTERLEAVED-NEXT: br label [[FOR_EXIT:%.*]] |
| ; CHECK-INTERLEAVED: for.exit: |
| ; CHECK-INTERLEAVED-NEXT: ret i32 [[TMP17]] |
| ; |
| ; CHECK-MAXBW-LABEL: define i32 @dotp( |
| ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; CHECK-MAXBW-NEXT: entry: |
| ; CHECK-MAXBW-NEXT: br label [[ENTRY:%.*]] |
| ; CHECK-MAXBW: vector.ph: |
| ; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() |
| ; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4 |
| ; CHECK-MAXBW-NEXT: br label [[FOR_BODY:%.*]] |
| ; CHECK-MAXBW: vector.body: |
| ; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] |
| ; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[ENTRY]] ], [ [[PARTIAL_REDUCE:%.*]], [[FOR_BODY]] ] |
| ; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]] |
| ; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP7]], align 1 |
| ; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[IV]] |
| ; CHECK-MAXBW-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP10]], align 1 |
| ; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32> |
| ; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32> |
| ; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = mul <vscale x 16 x i32> [[TMP4]], [[TMP5]] |
| ; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP6]]) |
| ; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP3]] |
| ; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = icmp eq i64 [[IV_NEXT]], 1024 |
| ; CHECK-MAXBW-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; CHECK-MAXBW: middle.block: |
| ; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]]) |
| ; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = sub i32 0, [[TMP9]] |
| ; CHECK-MAXBW-NEXT: br label [[FOR_EXIT:%.*]] |
| ; CHECK-MAXBW: for.exit: |
| ; CHECK-MAXBW-NEXT: ret i32 [[TMP11]] |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] |
| %accum = phi i32 [ 0, %entry ], [ %add, %for.body ] |
| %gep.a = getelementptr i8, ptr %a, i64 %iv |
| %load.a = load i8, ptr %gep.a, align 1 |
| %ext.a = zext i8 %load.a to i32 |
| %gep.b = getelementptr i8, ptr %b, i64 %iv |
| %load.b = load i8, ptr %gep.b, align 1 |
| %ext.b = zext i8 %load.b to i32 |
| %mul = mul i32 %ext.b, %ext.a |
| %add = sub i32 %accum, %mul |
| %iv.next = add i64 %iv, 1 |
| %exitcond.not = icmp eq i64 %iv.next, 1024 |
| br i1 %exitcond.not, label %for.exit, label %for.body |
| |
| for.exit: |
| ret i32 %add |
| } |
| |
| define i64 @partial_reduce_sub_sext_mul(ptr %x) #1 { |
| ; CHECK-INTERLEAVE1-LABEL: define i64 @partial_reduce_sub_sext_mul( |
| ; CHECK-INTERLEAVE1-SAME: ptr [[X:%.*]]) #[[ATTR1:[0-9]+]] { |
| ; CHECK-INTERLEAVE1-NEXT: entry: |
| ; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_PH:%.*]] |
| ; CHECK-INTERLEAVE1: vector.ph: |
| ; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_BODY:%.*]] |
| ; CHECK-INTERLEAVE1: vector.body: |
| ; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVE1-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[STRIDED_VEC:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1 |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = getelementptr [2 x i32], ptr [[X]], i64 [[TMP0]] |
| ; CHECK-INTERLEAVE1-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4 |
| ; CHECK-INTERLEAVE1-NEXT: [[STRIDED_VEC]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[STRIDED_VEC]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64> |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = sext <4 x i32> [[STRIDED_VEC]] to <4 x i64> |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = mul <4 x i64> [[TMP3]], [[TMP4]] |
| ; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v4i64(<2 x i64> [[VEC_PHI]], <4 x i64> [[TMP5]]) |
| ; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 36 |
| ; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] |
| ; CHECK-INTERLEAVE1: middle.block: |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[PARTIAL_REDUCE]]) |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = sub i64 0, [[TMP8]] |
| ; CHECK-INTERLEAVE1-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i64 3 |
| ; CHECK-INTERLEAVE1-NEXT: br label [[SCALAR_PH:%.*]] |
| ; CHECK-INTERLEAVE1: scalar.ph: |
| ; |
| ; CHECK-INTERLEAVED-LABEL: define i64 @partial_reduce_sub_sext_mul( |
| ; CHECK-INTERLEAVED-SAME: ptr [[X:%.*]]) #[[ATTR1:[0-9]+]] { |
| ; CHECK-INTERLEAVED-NEXT: entry: |
| ; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_PH:%.*]] |
| ; CHECK-INTERLEAVED: vector.ph: |
| ; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_BODY:%.*]] |
| ; CHECK-INTERLEAVED: vector.body: |
| ; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[STRIDED_VEC3:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE10:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[VEC_PHI2:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE11:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE12:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 4 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = add i64 [[INDEX]], 8 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = add i64 [[INDEX]], 12 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = add i64 [[TMP0]], 1 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP23:%.*]] = add i64 [[TMP16]], 1 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = add i64 [[TMP17]], 1 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr [2 x i32], ptr [[X]], i64 [[TMP1]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr [2 x i32], ptr [[X]], i64 [[TMP2]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP24:%.*]] = getelementptr [2 x i32], ptr [[X]], i64 [[TMP23]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP25:%.*]] = getelementptr [2 x i32], ptr [[X]], i64 [[TMP6]] |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP3]], align 4 |
| ; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_VEC2:%.*]] = load <8 x i32>, ptr [[TMP4]], align 4 |
| ; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_VEC6:%.*]] = load <8 x i32>, ptr [[TMP24]], align 4 |
| ; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <8 x i32> [[WIDE_VEC6]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_VEC8:%.*]] = load <8 x i32>, ptr [[TMP25]], align 4 |
| ; CHECK-INTERLEAVED-NEXT: [[STRIDED_VEC3]] = shufflevector <8 x i32> [[WIDE_VEC8]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[STRIDED_VEC]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP31:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> [[STRIDED_VEC5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC5]], <4 x i32> [[STRIDED_VEC7]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC7]], <4 x i32> [[STRIDED_VEC3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = sext <4 x i32> [[TMP5]] to <4 x i64> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = sext <4 x i32> [[STRIDED_VEC]] to <4 x i64> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = mul <4 x i64> [[TMP7]], [[TMP9]] |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v4i64(<2 x i64> [[VEC_PHI]], <4 x i64> [[TMP11]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = sext <4 x i32> [[TMP31]] to <4 x i64> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = sext <4 x i32> [[STRIDED_VEC5]] to <4 x i64> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = mul <4 x i64> [[TMP19]], [[TMP20]] |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE10]] = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v4i64(<2 x i64> [[VEC_PHI1]], <4 x i64> [[TMP21]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = sext <4 x i32> [[TMP13]] to <4 x i64> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = sext <4 x i32> [[STRIDED_VEC7]] to <4 x i64> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = mul <4 x i64> [[TMP8]], [[TMP10]] |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE11]] = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v4i64(<2 x i64> [[VEC_PHI2]], <4 x i64> [[TMP12]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP27:%.*]] = sext <4 x i32> [[TMP14]] to <4 x i64> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP28:%.*]] = sext <4 x i32> [[STRIDED_VEC3]] to <4 x i64> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP29:%.*]] = mul <4 x i64> [[TMP27]], [[TMP28]] |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE12]] = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v4i64(<2 x i64> [[VEC_PHI3]], <4 x i64> [[TMP29]]) |
| ; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32 |
| ; CHECK-INTERLEAVED-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] |
| ; CHECK-INTERLEAVED: middle.block: |
| ; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <2 x i64> [[PARTIAL_REDUCE10]], [[PARTIAL_REDUCE]] |
| ; CHECK-INTERLEAVED-NEXT: [[BIN_RDX13:%.*]] = add <2 x i64> [[PARTIAL_REDUCE11]], [[BIN_RDX]] |
| ; CHECK-INTERLEAVED-NEXT: [[BIN_RDX14:%.*]] = add <2 x i64> [[PARTIAL_REDUCE12]], [[BIN_RDX13]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP32:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[BIN_RDX14]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP30:%.*]] = sub i64 0, [[TMP32]] |
| ; CHECK-INTERLEAVED-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[STRIDED_VEC3]], i64 3 |
| ; CHECK-INTERLEAVED-NEXT: br label [[SCALAR_PH:%.*]] |
| ; CHECK-INTERLEAVED: scalar.ph: |
| ; |
| ; CHECK-MAXBW-LABEL: define i64 @partial_reduce_sub_sext_mul( |
| ; CHECK-MAXBW-SAME: ptr [[X:%.*]]) #[[ATTR1:[0-9]+]] { |
| ; CHECK-MAXBW-NEXT: entry: |
| ; CHECK-MAXBW-NEXT: br label [[VECTOR_PH:%.*]] |
| ; CHECK-MAXBW: vector.ph: |
| ; CHECK-MAXBW-NEXT: br label [[VECTOR_BODY:%.*]] |
| ; CHECK-MAXBW: vector.body: |
| ; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-MAXBW-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[STRIDED_VEC:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1 |
| ; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = getelementptr [2 x i32], ptr [[X]], i64 [[TMP0]] |
| ; CHECK-MAXBW-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4 |
| ; CHECK-MAXBW-NEXT: [[STRIDED_VEC]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> |
| ; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[STRIDED_VEC]], <4 x i32> <i32 3, i32 4, i32 5, i32 6> |
| ; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64> |
| ; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = sext <4 x i32> [[STRIDED_VEC]] to <4 x i64> |
| ; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = mul <4 x i64> [[TMP3]], [[TMP4]] |
| ; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v4i64(<2 x i64> [[VEC_PHI]], <4 x i64> [[TMP5]]) |
| ; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 |
| ; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 36 |
| ; CHECK-MAXBW-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] |
| ; CHECK-MAXBW: middle.block: |
| ; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[PARTIAL_REDUCE]]) |
| ; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = sub i64 0, [[TMP8]] |
| ; CHECK-MAXBW-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[STRIDED_VEC]], i64 3 |
| ; CHECK-MAXBW-NEXT: br label [[SCALAR_PH:%.*]] |
| ; CHECK-MAXBW: scalar.ph: |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %prev = phi i32 [ 0, %entry ], [ %load, %loop ] |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %accum = phi i64 [ 0, %entry ], [ %sub, %loop ] |
| %iv.next = add i64 %iv, 1 |
| %gep = getelementptr [2 x i32], ptr %x, i64 %iv.next |
| %load = load i32, ptr %gep, align 4 |
| %ext.prev = sext i32 %prev to i64 |
| %ext.load = sext i32 %load to i64 |
| %mul = mul i64 %ext.prev, %ext.load |
| %sub = sub i64 %accum, %mul |
| %exitcond.not = icmp eq i64 %iv, 38 |
| br i1 %exitcond.not, label %exit, label %loop |
| |
| exit: |
| ret i64 %sub |
| } |
| |
| define float @fdotp_fsub(ptr %a, ptr %b, ptr %c) #2 { |
| ; CHECK-INTERLEAVE1-LABEL: define float @fdotp_fsub( |
| ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR2:[0-9]+]] { |
| ; CHECK-INTERLEAVE1-NEXT: entry: |
| ; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_PH:%.*]] |
| ; CHECK-INTERLEAVE1: vector.ph: |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3 |
| ; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_BODY:%.*]] |
| ; CHECK-INTERLEAVE1: vector.body: |
| ; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ insertelement (<vscale x 4 x float> splat (float -0.000000e+00), float -0.000000e+00, i64 0), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]] |
| ; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP0]], align 2 |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]] |
| ; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x half>, ptr [[TMP1]], align 2 |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = getelementptr half, ptr [[C]], i64 [[INDEX]] |
| ; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 8 x half>, ptr [[TMP8]], align 2 |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD1]] to <vscale x 8 x float> |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD]] to <vscale x 8 x float> |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = fmul reassoc contract <vscale x 8 x float> [[TMP10]], [[TMP11]] |
| ; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE:%.*]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[VEC_PHI]], <vscale x 8 x float> [[TMP12]]) |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP13:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD2]] to <vscale x 8 x float> |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = fmul reassoc contract <vscale x 8 x float> [[TMP13]], [[TMP11]] |
| ; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE3]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[PARTIAL_REDUCE]], <vscale x 8 x float> [[TMP9]]) |
| ; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 |
| ; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] |
| ; CHECK-INTERLEAVE1: middle.block: |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[PARTIAL_REDUCE3]]) |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = fsub float 0.000000e+00, [[TMP6]] |
| ; CHECK-INTERLEAVE1-NEXT: br label [[FOR_EXIT:%.*]] |
| ; CHECK-INTERLEAVE1: for.exit: |
| ; CHECK-INTERLEAVE1-NEXT: ret float [[TMP7]] |
| ; |
| ; CHECK-INTERLEAVED-LABEL: define float @fdotp_fsub( |
| ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR2:[0-9]+]] { |
| ; CHECK-INTERLEAVED-NEXT: entry: |
| ; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_PH:%.*]] |
| ; CHECK-INTERLEAVED: vector.ph: |
| ; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() |
| ; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP3]], 3 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP3]], 4 |
| ; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_BODY:%.*]] |
| ; CHECK-INTERLEAVED: vector.body: |
| ; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ insertelement (<vscale x 4 x float> splat (float -0.000000e+00), float -0.000000e+00, i64 0), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x float> [ insertelement (<vscale x 4 x float> splat (float -0.000000e+00), float -0.000000e+00, i64 0), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE9:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr half, ptr [[TMP0]], i64 [[TMP1]] |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP0]], align 2 |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 8 x half>, ptr [[TMP4]], align 2 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr half, ptr [[TMP2]], i64 [[TMP1]] |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 8 x half>, ptr [[TMP2]], align 2 |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 8 x half>, ptr [[TMP6]], align 2 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = getelementptr half, ptr [[C]], i64 [[INDEX]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr half, ptr [[TMP16]], i64 [[TMP1]] |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 8 x half>, ptr [[TMP16]], align 2 |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 8 x half>, ptr [[TMP8]], align 2 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD3]] to <vscale x 8 x float> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD]] to <vscale x 8 x float> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = fmul reassoc contract <vscale x 8 x float> [[TMP9]], [[TMP19]] |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE:%.*]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[VEC_PHI]], <vscale x 8 x float> [[TMP20]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD4]] to <vscale x 8 x float> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD2]] to <vscale x 8 x float> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = fmul reassoc contract <vscale x 8 x float> [[TMP21]], [[TMP13]] |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE7:%.*]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[VEC_PHI1]], <vscale x 8 x float> [[TMP14]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD5]] to <vscale x 8 x float> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = fmul reassoc contract <vscale x 8 x float> [[TMP15]], [[TMP19]] |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE8]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[PARTIAL_REDUCE]], <vscale x 8 x float> [[TMP22]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD6]] to <vscale x 8 x float> |
| ; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = fmul reassoc contract <vscale x 8 x float> [[TMP17]], [[TMP13]] |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE9]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[PARTIAL_REDUCE7]], <vscale x 8 x float> [[TMP18]]) |
| ; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 |
| ; CHECK-INTERLEAVED-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] |
| ; CHECK-INTERLEAVED: middle.block: |
| ; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = fadd reassoc contract <vscale x 4 x float> [[PARTIAL_REDUCE9]], [[PARTIAL_REDUCE8]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[BIN_RDX]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = fsub float 0.000000e+00, [[TMP11]] |
| ; CHECK-INTERLEAVED-NEXT: br label [[FOR_EXIT:%.*]] |
| ; CHECK-INTERLEAVED: for.exit: |
| ; CHECK-INTERLEAVED-NEXT: ret float [[TMP12]] |
| ; |
| ; CHECK-MAXBW-LABEL: define float @fdotp_fsub( |
| ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) #[[ATTR2:[0-9]+]] { |
| ; CHECK-MAXBW-NEXT: entry: |
| ; CHECK-MAXBW-NEXT: br label [[VECTOR_PH:%.*]] |
| ; CHECK-MAXBW: vector.ph: |
| ; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() |
| ; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3 |
| ; CHECK-MAXBW-NEXT: br label [[VECTOR_BODY:%.*]] |
| ; CHECK-MAXBW: vector.body: |
| ; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ insertelement (<vscale x 4 x float> splat (float -0.000000e+00), float -0.000000e+00, i64 0), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]] |
| ; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP0]], align 2 |
| ; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]] |
| ; CHECK-MAXBW-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x half>, ptr [[TMP1]], align 2 |
| ; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = getelementptr half, ptr [[C]], i64 [[INDEX]] |
| ; CHECK-MAXBW-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 8 x half>, ptr [[TMP8]], align 2 |
| ; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD1]] to <vscale x 8 x float> |
| ; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD]] to <vscale x 8 x float> |
| ; CHECK-MAXBW-NEXT: [[TMP12:%.*]] = fmul reassoc contract <vscale x 8 x float> [[TMP10]], [[TMP11]] |
| ; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE:%.*]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[VEC_PHI]], <vscale x 8 x float> [[TMP12]]) |
| ; CHECK-MAXBW-NEXT: [[TMP13:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD2]] to <vscale x 8 x float> |
| ; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = fmul reassoc contract <vscale x 8 x float> [[TMP13]], [[TMP11]] |
| ; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE3]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[PARTIAL_REDUCE]], <vscale x 8 x float> [[TMP9]]) |
| ; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] |
| ; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 |
| ; CHECK-MAXBW-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] |
| ; CHECK-MAXBW: middle.block: |
| ; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[PARTIAL_REDUCE3]]) |
| ; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = fsub float 0.000000e+00, [[TMP6]] |
| ; CHECK-MAXBW-NEXT: br label [[FOR_EXIT:%.*]] |
| ; CHECK-MAXBW: for.exit: |
| ; CHECK-MAXBW-NEXT: ret float [[TMP7]] |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] |
| %accum = phi float [ 0.0, %entry ], [ %sub, %for.body ] |
| %gep.a = getelementptr half, ptr %a, i64 %iv |
| %load.a = load half, ptr %gep.a, align 2 |
| %ext.a = fpext half %load.a to float |
| %gep.b = getelementptr half, ptr %b, i64 %iv |
| %load.b = load half, ptr %gep.b, align 2 |
| %ext.b = fpext half %load.b to float |
| %gep.c = getelementptr half, ptr %c, i64 %iv |
| %load.c = load half, ptr %gep.c, align 2 |
| %ext.c = fpext half %load.c to float |
| %mul.ab = fmul reassoc contract float %ext.b, %ext.a |
| %mul.ac = fmul reassoc contract float %ext.c, %ext.a |
| %sub.ab = fsub reassoc contract float %accum, %mul.ab |
| %sub = fsub reassoc contract float %sub.ab, %mul.ac |
| %iv.next = add i64 %iv, 1 |
| %exitcond.not = icmp eq i64 %iv.next, 1024 |
| br i1 %exitcond.not, label %for.exit, label %for.body |
| |
| for.exit: |
| ret float %sub |
| } |
| |
| define float @reduce_fsub(ptr %a) #1 { |
| ; CHECK-INTERLEAVE1-LABEL: define float @reduce_fsub( |
| ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR1]] { |
| ; CHECK-INTERLEAVE1-NEXT: entry: |
| ; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_PH:%.*]] |
| ; CHECK-INTERLEAVE1: vector.ph: |
| ; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_BODY:%.*]] |
| ; CHECK-INTERLEAVE1: vector.body: |
| ; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]] |
| ; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 1 |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float> |
| ; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP1]]) |
| ; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 |
| ; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] |
| ; CHECK-INTERLEAVE1: middle.block: |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PARTIAL_REDUCE]]) |
| ; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = fsub float -0.000000e+00, [[TMP3]] |
| ; CHECK-INTERLEAVE1-NEXT: br label [[SCALAR_PH:%.*]] |
| ; CHECK-INTERLEAVE1: scalar.ph: |
| ; |
| ; CHECK-INTERLEAVED-LABEL: define float @reduce_fsub( |
| ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR1]] { |
| ; CHECK-INTERLEAVED-NEXT: entry: |
| ; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_PH:%.*]] |
| ; CHECK-INTERLEAVED: vector.ph: |
| ; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_BODY:%.*]] |
| ; CHECK-INTERLEAVED: vector.body: |
| ; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[VEC_PHI2:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[VEC_PHI3:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE9:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr half, ptr [[TMP0]], i64 8 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr half, ptr [[TMP0]], i64 16 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr half, ptr [[TMP0]], i64 24 |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 1 |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x half>, ptr [[TMP1]], align 1 |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x half>, ptr [[TMP2]], align 1 |
| ; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x half>, ptr [[TMP3]], align 1 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float> |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP4]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = fpext <8 x half> [[WIDE_LOAD4]] to <8 x float> |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE7]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI1]], <8 x float> [[TMP5]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = fpext <8 x half> [[WIDE_LOAD5]] to <8 x float> |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE8]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI2]], <8 x float> [[TMP6]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = fpext <8 x half> [[WIDE_LOAD6]] to <8 x float> |
| ; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE9]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI3]], <8 x float> [[TMP7]]) |
| ; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32 |
| ; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 |
| ; CHECK-INTERLEAVED-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] |
| ; CHECK-INTERLEAVED: middle.block: |
| ; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = fadd reassoc contract <4 x float> [[PARTIAL_REDUCE7]], [[PARTIAL_REDUCE]] |
| ; CHECK-INTERLEAVED-NEXT: [[BIN_RDX10:%.*]] = fadd reassoc contract <4 x float> [[PARTIAL_REDUCE8]], [[BIN_RDX]] |
| ; CHECK-INTERLEAVED-NEXT: [[BIN_RDX11:%.*]] = fadd reassoc contract <4 x float> [[PARTIAL_REDUCE9]], [[BIN_RDX10]] |
| ; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[BIN_RDX11]]) |
| ; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = fsub float -0.000000e+00, [[TMP9]] |
| ; CHECK-INTERLEAVED-NEXT: br label [[SCALAR_PH:%.*]] |
| ; CHECK-INTERLEAVED: scalar.ph: |
| ; |
| ; CHECK-MAXBW-LABEL: define float @reduce_fsub( |
| ; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR1]] { |
| ; CHECK-MAXBW-NEXT: entry: |
| ; CHECK-MAXBW-NEXT: br label [[VECTOR_PH:%.*]] |
| ; CHECK-MAXBW: vector.ph: |
| ; CHECK-MAXBW-NEXT: br label [[VECTOR_BODY:%.*]] |
| ; CHECK-MAXBW: vector.body: |
| ; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ] |
| ; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]] |
| ; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <8 x half>, ptr [[TMP0]], align 1 |
| ; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = fpext <8 x half> [[WIDE_LOAD]] to <8 x float> |
| ; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call reassoc contract <4 x float> @llvm.vector.partial.reduce.fadd.v4f32.v8f32(<4 x float> [[VEC_PHI]], <8 x float> [[TMP1]]) |
| ; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 |
| ; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 |
| ; CHECK-MAXBW-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] |
| ; CHECK-MAXBW: middle.block: |
| ; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PARTIAL_REDUCE]]) |
| ; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = fsub float -0.000000e+00, [[TMP3]] |
| ; CHECK-MAXBW-NEXT: br label [[SCALAR_PH:%.*]] |
| ; CHECK-MAXBW: scalar.ph: |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] |
| %accum = phi float [ -0.0, %entry ], [ %sub, %for.body ] |
| %gep.a = getelementptr half, ptr %a, i64 %iv |
| %load.a = load half, ptr %gep.a, align 1 |
| %ext.a = fpext half %load.a to float |
| %sub = fsub reassoc contract float %accum, %ext.a |
| %iv.next = add i64 %iv, 1 |
| %exitcond.not = icmp eq i64 %iv.next, 1025 |
| br i1 %exitcond.not, label %for.exit, label %for.body |
| |
| for.exit: |
| ret float %sub |
| } |
| |
| !7 = distinct !{!7, !8, !9, !10} |
| !8 = !{!"llvm.loop.mustprogress"} |
| !9 = !{!"llvm.loop.vectorize.predicate.enable"} |
| !10 = !{!"llvm.loop.vectorize.enable"} |
| |
| attributes #0 = { vscale_range(1,16) "target-features"="+sve" } |
| attributes #1 = { "target-cpu"="apple-m1" } |
| attributes #2 = { vscale_range(1,16) "target-features"="+sve2" } |
