| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 6 | |
| ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -scalable-vectorization=on \ | |
| ; RUN: -force-target-supports-masked-memory-ops -force-target-supports-scalable-vectors \ | |
| ; RUN: -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses -force-target-supports-gather-scatter-ops \ | |
| ; RUN: -S < %s | FileCheck %s --check-prefix=CHECK-GATHER-ENABLED | |
| ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -scalable-vectorization=on \ | |
| ; RUN: -force-target-supports-masked-memory-ops -force-target-supports-scalable-vectors \ | |
| ; RUN: -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses \ | |
| ; RUN: -S < %s | FileCheck %s --check-prefix=CHECK-GATHER-DISABLED | |
| ; Test whether the legality of gather/scatter has an impact on if-conversion | |
| ; for scalable VFs. In either case, the example below should be vectorized with | |
| ; VF = vscale x 8 and a masked deinterleaved load. | |
| ; Loop with two conditional (interleaved) loads. | |
| define void @conv_interleaved_loads_load(ptr noalias %a, ptr readonly %b, ptr readonly %c, ptr readonly %flags, i64 %cond) { | |
| ; CHECK-GATHER-ENABLED-LABEL: define void @conv_interleaved_loads_load( | |
| ; CHECK-GATHER-ENABLED-SAME: ptr noalias [[A:%.*]], ptr readonly [[B:%.*]], ptr readonly [[C:%.*]], ptr readonly [[FLAGS:%.*]], i64 [[COND:%.*]]) { | |
| ; CHECK-GATHER-ENABLED-NEXT: [[ENTRY:.*:]] | |
| ; CHECK-GATHER-ENABLED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() | |
| ; CHECK-GATHER-ENABLED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3 | |
| ; CHECK-GATHER-ENABLED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]] | |
| ; CHECK-GATHER-ENABLED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK-GATHER-ENABLED: [[VECTOR_PH]]: | |
| ; CHECK-GATHER-ENABLED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]] | |
| ; CHECK-GATHER-ENABLED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] | |
| ; CHECK-GATHER-ENABLED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[COND]], i64 0 | |
| ; CHECK-GATHER-ENABLED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer | |
| ; CHECK-GATHER-ENABLED-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK-GATHER-ENABLED: [[VECTOR_BODY]]: | |
| ; CHECK-GATHER-ENABLED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-GATHER-ENABLED-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[INDEX]] | |
| ; CHECK-GATHER-ENABLED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 2 | |
| ; CHECK-GATHER-ENABLED-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[FLAGS]], i64 [[INDEX]] | |
| ; CHECK-GATHER-ENABLED-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8 | |
| ; CHECK-GATHER-ENABLED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 8 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]] | |
| ; CHECK-GATHER-ENABLED-NEXT: [[TMP7:%.*]] = getelementptr [2 x i16], ptr [[C]], i64 [[INDEX]], i64 0 | |
| ; CHECK-GATHER-ENABLED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> [[TMP6]], <vscale x 8 x i1> [[TMP6]]) | |
| ; CHECK-GATHER-ENABLED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.masked.load.nxv16i16.p0(ptr align 2 [[TMP7]], <vscale x 16 x i1> [[INTERLEAVED_MASK]], <vscale x 16 x i16> poison) | |
| ; CHECK-GATHER-ENABLED-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave2.nxv16i16(<vscale x 16 x i16> [[WIDE_MASKED_VEC]]) | |
| ; CHECK-GATHER-ENABLED-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 0 | |
| ; CHECK-GATHER-ENABLED-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 1 | |
| ; CHECK-GATHER-ENABLED-NEXT: [[TMP10:%.*]] = add <vscale x 8 x i16> [[TMP8]], [[TMP9]] | |
| ; CHECK-GATHER-ENABLED-NEXT: [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP6]], <vscale x 8 x i16> [[TMP10]], <vscale x 8 x i16> zeroinitializer | |
| ; CHECK-GATHER-ENABLED-NEXT: [[TMP11:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD]], [[PREDPHI]] | |
| ; CHECK-GATHER-ENABLED-NEXT: [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[INDEX]] | |
| ; CHECK-GATHER-ENABLED-NEXT: store <vscale x 8 x i16> [[TMP11]], ptr [[TMP12]], align 2 | |
| ; CHECK-GATHER-ENABLED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]] | |
| ; CHECK-GATHER-ENABLED-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-GATHER-ENABLED-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] | |
| ; CHECK-GATHER-ENABLED: [[MIDDLE_BLOCK]]: | |
| ; CHECK-GATHER-ENABLED-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]] | |
| ; CHECK-GATHER-ENABLED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] | |
| ; CHECK-GATHER-ENABLED: [[SCALAR_PH]]: | |
| ; | |
| ; CHECK-GATHER-DISABLED-LABEL: define void @conv_interleaved_loads_load( | |
| ; CHECK-GATHER-DISABLED-SAME: ptr noalias [[A:%.*]], ptr readonly [[B:%.*]], ptr readonly [[C:%.*]], ptr readonly [[FLAGS:%.*]], i64 [[COND:%.*]]) { | |
| ; CHECK-GATHER-DISABLED-NEXT: [[ENTRY:.*:]] | |
| ; CHECK-GATHER-DISABLED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() | |
| ; CHECK-GATHER-DISABLED-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP2]], 3 | |
| ; CHECK-GATHER-DISABLED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP5]] | |
| ; CHECK-GATHER-DISABLED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[EXIT:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK-GATHER-DISABLED: [[VECTOR_PH]]: | |
| ; CHECK-GATHER-DISABLED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP5]] | |
| ; CHECK-GATHER-DISABLED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] | |
| ; CHECK-GATHER-DISABLED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[COND]], i64 0 | |
| ; CHECK-GATHER-DISABLED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer | |
| ; CHECK-GATHER-DISABLED-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK-GATHER-DISABLED: [[VECTOR_BODY]]: | |
| ; CHECK-GATHER-DISABLED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-GATHER-DISABLED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[INDEX]] | |
| ; CHECK-GATHER-DISABLED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP0]], align 2 | |
| ; CHECK-GATHER-DISABLED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[FLAGS]], i64 [[INDEX]] | |
| ; CHECK-GATHER-DISABLED-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i64>, ptr [[TMP1]], align 8 | |
| ; CHECK-GATHER-DISABLED-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 8 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]] | |
| ; CHECK-GATHER-DISABLED-NEXT: [[TMP3:%.*]] = getelementptr [2 x i16], ptr [[C]], i64 [[INDEX]], i64 0 | |
| ; CHECK-GATHER-DISABLED-NEXT: [[INTERLEAVED_MASK:%.*]] = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> [[TMP4]], <vscale x 8 x i1> [[TMP4]]) | |
| ; CHECK-GATHER-DISABLED-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <vscale x 16 x i16> @llvm.masked.load.nxv16i16.p0(ptr align 2 [[TMP3]], <vscale x 16 x i1> [[INTERLEAVED_MASK]], <vscale x 16 x i16> poison) | |
| ; CHECK-GATHER-DISABLED-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave2.nxv16i16(<vscale x 16 x i16> [[WIDE_MASKED_VEC]]) | |
| ; CHECK-GATHER-DISABLED-NEXT: [[TMP10:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 0 | |
| ; CHECK-GATHER-DISABLED-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[STRIDED_VEC]], 1 | |
| ; CHECK-GATHER-DISABLED-NEXT: [[TMP8:%.*]] = add <vscale x 8 x i16> [[TMP10]], [[TMP7]] | |
| ; CHECK-GATHER-DISABLED-NEXT: [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP4]], <vscale x 8 x i16> [[TMP8]], <vscale x 8 x i16> zeroinitializer | |
| ; CHECK-GATHER-DISABLED-NEXT: [[TMP9:%.*]] = add <vscale x 8 x i16> [[WIDE_LOAD]], [[PREDPHI]] | |
| ; CHECK-GATHER-DISABLED-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[INDEX]] | |
| ; CHECK-GATHER-DISABLED-NEXT: store <vscale x 8 x i16> [[TMP9]], ptr [[TMP6]], align 2 | |
| ; CHECK-GATHER-DISABLED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] | |
| ; CHECK-GATHER-DISABLED-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-GATHER-DISABLED-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] | |
| ; CHECK-GATHER-DISABLED: [[MIDDLE_BLOCK]]: | |
| ; CHECK-GATHER-DISABLED-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]] | |
| ; CHECK-GATHER-DISABLED-NEXT: br i1 [[CMP_N]], [[EXIT1:label %.*]], label %[[EXIT]] | |
| ; CHECK-GATHER-DISABLED: [[EXIT]]: | |
| ; | |
| entry: | |
| br label %for.body | |
| for.body: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] | |
| %arrayidx0 = getelementptr inbounds i16, ptr %b, i64 %iv | |
| %0 = load i16, ptr %arrayidx0, align 2 | |
| %flagidx = getelementptr inbounds i64, ptr %flags, i64 %iv | |
| %val.for.cmp = load i64, ptr %flagidx, align 8 | |
| %do.access = icmp ne i64 %val.for.cmp, %cond | |
| br i1 %do.access, label %if.then, label %for.inc | |
| if.then: | |
| %arrayidx1 = getelementptr inbounds [2 x i16], ptr %c, i64 %iv, i64 0 | |
| %cond.load.1 = load i16, ptr %arrayidx1, align 2 | |
| %arrayidx2 = getelementptr inbounds [2 x i16], ptr %c, i64 %iv, i64 1 | |
| %cond.load.2 = load i16, ptr %arrayidx2, align 2 | |
| %cond.loads.sum = add i16 %cond.load.1, %cond.load.2 | |
| br label %for.inc | |
| for.inc: | |
| %1 = phi i16 [ zeroinitializer, %for.body ], [ %cond.loads.sum, %if.then ] | |
| %result = add i16 %0, %1 | |
| %arrayidx3 = getelementptr inbounds i16, ptr %a, i64 %iv | |
| store i16 %result, ptr %arrayidx3, align 2 | |
| %iv.next = add nuw nsw i64 %iv, 1 | |
| %exitcond.not = icmp eq i64 %iv.next, 1024 | |
| br i1 %exitcond.not, label %exit, label %for.body | |
| exit: | |
| ret void | |
| } |