| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6 |
| ; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path < %s | FileCheck %s |
| |
| ; Test coverage for contiguous access detection in outer loop vectorization. |
| ; Tests various stride and type combinations. |
| |
| ; --- stride-1 float load --- |
| ; for (i = 0; i < N; i++) |
| ; for (j = 0; j < M; j++) |
| ; A[i * M + j] *= scale[i]; |
| ; |
| ; scale[i] is a stride-1 access w.r.t. the outer loop IV. |
| define void @scale_rows(ptr noalias %A, ptr noalias %scale, i64 %N, i64 %M) { |
| ; CHECK-LABEL: define void @scale_rows( |
| ; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[SCALE:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ] |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ] |
| ; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[SCALE]], <4 x i64> [[VEC_IND]] |
| ; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x ptr> [[WIDE_GEP]], i64 0 |
| ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4 |
| ; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: br label %[[INNER_BODY1:.*]] |
| ; CHECK: [[INNER_BODY1]]: |
| ; CHECK-NEXT: [[J2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[INNER_BODY1]] ] |
| ; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i64> [[TMP2]], [[J2]] |
| ; CHECK-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP3]] |
| ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP3]], <4 x i1> splat (i1 true), <4 x float> poison) |
| ; CHECK-NEXT: [[TMP4:%.*]] = fmul <4 x float> [[WIDE_MASKED_GATHER]], [[WIDE_LOAD]] |
| ; CHECK-NEXT: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> [[TMP4]], <4 x ptr> align 4 [[WIDE_GEP3]], <4 x i1> splat (i1 true)) |
| ; CHECK-NEXT: [[TMP5]] = add nuw nsw <4 x i64> [[J2]], splat (i64 1) |
| ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0 |
| ; CHECK-NEXT: br i1 [[TMP7]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]] |
| ; CHECK: [[OUTER_LATCH4]]: |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) |
| ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; |
| entry: |
| br label %outer.header |
| |
| outer.header: |
| %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ] |
| %scale.ptr = getelementptr inbounds float, ptr %scale, i64 %i |
| %scale.val = load float, ptr %scale.ptr, align 4 |
| %i.mul.M = mul nsw i64 %i, %M |
| br label %inner.body |
| |
| inner.body: |
| %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ] |
| %idx = add nsw i64 %i.mul.M, %j |
| %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx |
| %A.val = load float, ptr %A.ptr, align 4 |
| %mul = fmul float %A.val, %scale.val |
| store float %mul, ptr %A.ptr, align 4 |
| %j.next = add nuw nsw i64 %j, 1 |
| %j.cmp = icmp eq i64 %j.next, %M |
| br i1 %j.cmp, label %outer.latch, label %inner.body |
| |
| outer.latch: |
| %i.next = add nuw nsw i64 %i, 1 |
| %i.cmp = icmp eq i64 %i.next, %N |
| br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0 |
| |
| exit: |
| ret void |
| } |
| |
| ; --- stride-2 float load (NOT contiguous) --- |
| ; for (i = 0; i < N; i++) |
| ; for (j = 0; j < M; j++) |
| ; A[i*M+j] *= scale[2*i]; |
| ; |
| ; scale[2*i] has stride 2*sizeof(float) = 8 bytes, not matching |
| ; sizeof(float) = 4. Should remain a gather. |
| define void @stride2_float_load(ptr noalias %A, ptr noalias %scale, i64 %N, i64 %M) { |
| ; CHECK-LABEL: define void @stride2_float_load( |
| ; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[SCALE:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH5:.*]] ] |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH5]] ] |
| ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw <4 x i64> [[VEC_IND]], splat (i64 1) |
| ; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds float, ptr [[SCALE]], <4 x i64> [[TMP1]] |
| ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x float> poison) |
| ; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: br label %[[INNER_BODY1:.*]] |
| ; CHECK: [[INNER_BODY1]]: |
| ; CHECK-NEXT: [[J2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[INNER_BODY1]] ] |
| ; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i64> [[TMP2]], [[J2]] |
| ; CHECK-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP3]] |
| ; CHECK-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[WIDE_GEP3]], <4 x i1> splat (i1 true), <4 x float> poison) |
| ; CHECK-NEXT: [[TMP4:%.*]] = fmul <4 x float> [[WIDE_MASKED_GATHER4]], [[WIDE_MASKED_GATHER]] |
| ; CHECK-NEXT: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> [[TMP4]], <4 x ptr> align 4 [[WIDE_GEP3]], <4 x i1> splat (i1 true)) |
| ; CHECK-NEXT: [[TMP5]] = add nuw nsw <4 x i64> [[J2]], splat (i64 1) |
| ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0 |
| ; CHECK-NEXT: br i1 [[TMP7]], label %[[OUTER_LATCH5]], label %[[INNER_BODY1]] |
| ; CHECK: [[OUTER_LATCH5]]: |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) |
| ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; |
| entry: |
| br label %outer.header |
| |
| outer.header: |
| %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ] |
| %i2 = shl nuw nsw i64 %i, 1 |
| %scale.ptr = getelementptr inbounds float, ptr %scale, i64 %i2 |
| %scale.val = load float, ptr %scale.ptr, align 4 |
| %i.mul.M = mul nsw i64 %i, %M |
| br label %inner.body |
| |
| inner.body: |
| %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ] |
| %idx = add nsw i64 %i.mul.M, %j |
| %A.ptr = getelementptr inbounds float, ptr %A, i64 %idx |
| %A.val = load float, ptr %A.ptr, align 4 |
| %mul = fmul float %A.val, %scale.val |
| store float %mul, ptr %A.ptr, align 4 |
| %j.next = add nuw nsw i64 %j, 1 |
| %j.cmp = icmp eq i64 %j.next, %M |
| br i1 %j.cmp, label %outer.latch, label %inner.body |
| |
| outer.latch: |
| %i.next = add nuw nsw i64 %i, 1 |
| %i.cmp = icmp eq i64 %i.next, %N |
| br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0 |
| |
| exit: |
| ret void |
| } |
| |
| ; --- stride-1 double load --- |
| ; for (i = 0; i < N; i++) |
| ; for (j = 0; j < M; j++) |
| ; A[i*M+j] *= scale[i]; // double type |
| ; |
| ; scale[i] has stride sizeof(double) = 8, matching the element type. |
| define void @stride1_double_load(ptr noalias %A, ptr noalias %scale, i64 %N, i64 %M) { |
| ; CHECK-LABEL: define void @stride1_double_load( |
| ; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[SCALE:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ] |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ] |
| ; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds double, ptr [[SCALE]], <4 x i64> [[VEC_IND]] |
| ; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x ptr> [[WIDE_GEP]], i64 0 |
| ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP1]], align 8 |
| ; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: br label %[[INNER_BODY1:.*]] |
| ; CHECK: [[INNER_BODY1]]: |
| ; CHECK-NEXT: [[J2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[INNER_BODY1]] ] |
| ; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i64> [[TMP2]], [[J2]] |
| ; CHECK-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds double, ptr [[A]], <4 x i64> [[TMP3]] |
| ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[WIDE_GEP3]], <4 x i1> splat (i1 true), <4 x double> poison) |
| ; CHECK-NEXT: [[TMP4:%.*]] = fmul <4 x double> [[WIDE_MASKED_GATHER]], [[WIDE_LOAD]] |
| ; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP4]], <4 x ptr> align 8 [[WIDE_GEP3]], <4 x i1> splat (i1 true)) |
| ; CHECK-NEXT: [[TMP5]] = add nuw nsw <4 x i64> [[J2]], splat (i64 1) |
| ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0 |
| ; CHECK-NEXT: br i1 [[TMP7]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]] |
| ; CHECK: [[OUTER_LATCH4]]: |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) |
| ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; |
| entry: |
| br label %outer.header |
| |
| outer.header: |
| %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ] |
| %scale.ptr = getelementptr inbounds double, ptr %scale, i64 %i |
| %scale.val = load double, ptr %scale.ptr, align 8 |
| %i.mul.M = mul nsw i64 %i, %M |
| br label %inner.body |
| |
| inner.body: |
| %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ] |
| %idx = add nsw i64 %i.mul.M, %j |
| %A.ptr = getelementptr inbounds double, ptr %A, i64 %idx |
| %A.val = load double, ptr %A.ptr, align 8 |
| %mul = fmul double %A.val, %scale.val |
| store double %mul, ptr %A.ptr, align 8 |
| %j.next = add nuw nsw i64 %j, 1 |
| %j.cmp = icmp eq i64 %j.next, %M |
| br i1 %j.cmp, label %outer.latch, label %inner.body |
| |
| outer.latch: |
| %i.next = add nuw nsw i64 %i, 1 |
| %i.cmp = icmp eq i64 %i.next, %N |
| br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0 |
| |
| exit: |
| ret void |
| } |
| |
| ; --- stride-1 i1 load and store (NOT contiguous) --- |
| ; for (i = 0; i < N; i++) { |
| ; bool b = flags[i]; |
| ; for (j = 0; j < M; j++) |
| ; A[i*M+j] = b; |
| ; flags[i] = !b; |
| ; } |
| ; |
| ; flags[i] has stride 1 in units of i1, but i1 is bit-packed in vectors: 4 |
| ; consecutive i1 scalars span 4 bytes, while a <4 x i1> access covers a single |
| ; byte. Should remain a gather and scatter. |
| define void @stride1_i1_load_store(ptr noalias %A, ptr noalias %flags, i64 %N, i64 %M) { |
| ; CHECK-LABEL: define void @stride1_i1_load_store( |
| ; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[FLAGS:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ] |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ] |
| ; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i1, ptr [[FLAGS]], <4 x i64> [[VEC_IND]] |
| ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i1> @llvm.masked.gather.v4i1.v4p0(<4 x ptr> align 1 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i1> poison) |
| ; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i1> [[WIDE_LOAD]] to <4 x i8> |
| ; CHECK-NEXT: br label %[[INNER_BODY1:.*]] |
| ; CHECK: [[INNER_BODY1]]: |
| ; CHECK-NEXT: [[J2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[INNER_BODY1]] ] |
| ; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i64> [[TMP2]], [[J2]] |
| ; CHECK-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds i8, ptr [[A]], <4 x i64> [[TMP4]] |
| ; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP3]], <4 x ptr> align 1 [[WIDE_GEP3]], <4 x i1> splat (i1 true)) |
| ; CHECK-NEXT: [[TMP5]] = add nuw nsw <4 x i64> [[J2]], splat (i64 1) |
| ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0 |
| ; CHECK-NEXT: br i1 [[TMP7]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]] |
| ; CHECK: [[OUTER_LATCH4]]: |
| ; CHECK-NEXT: [[TMP8:%.*]] = xor <4 x i1> [[WIDE_LOAD]], splat (i1 true) |
| ; CHECK-NEXT: call void @llvm.masked.scatter.v4i1.v4p0(<4 x i1> [[TMP8]], <4 x ptr> align 1 [[WIDE_GEP]], <4 x i1> splat (i1 true)) |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) |
| ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; |
| entry: |
| br label %outer.header |
| |
| outer.header: |
| %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ] |
| %flags.ptr = getelementptr inbounds i1, ptr %flags, i64 %i |
| %flag = load i1, ptr %flags.ptr, align 1 |
| %i.mul.M = mul nsw i64 %i, %M |
| %flag.ext = zext i1 %flag to i8 |
| br label %inner.body |
| |
| inner.body: |
| %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ] |
| %idx = add nsw i64 %i.mul.M, %j |
| %A.ptr = getelementptr inbounds i8, ptr %A, i64 %idx |
| store i8 %flag.ext, ptr %A.ptr, align 1 |
| %j.next = add nuw nsw i64 %j, 1 |
| %j.cmp = icmp eq i64 %j.next, %M |
| br i1 %j.cmp, label %outer.latch, label %inner.body |
| |
| outer.latch: |
| %flag.not = xor i1 %flag, true |
| store i1 %flag.not, ptr %flags.ptr, align 1 |
| %i.next = add nuw nsw i64 %i, 1 |
| %i.cmp = icmp eq i64 %i.next, %N |
| br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0 |
| |
| exit: |
| ret void |
| } |
| |
| ; --- stride-1 i4 load and store (NOT contiguous) --- |
| ; Same as above with i4, which is also bit-packed in vectors: 4 consecutive i4 |
| ; scalars span 4 bytes, while a <4 x i4> access covers 2 bytes. |
| define void @stride1_i4_load_store(ptr noalias %A, ptr noalias %vals, i64 %N, i64 %M) { |
| ; CHECK-LABEL: define void @stride1_i4_load_store( |
| ; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[VALS:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ] |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ] |
| ; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i4, ptr [[VALS]], <4 x i64> [[VEC_IND]] |
| ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i4> @llvm.masked.gather.v4i4.v4p0(<4 x ptr> align 1 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i4> poison) |
| ; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i4> [[WIDE_LOAD]] to <4 x i8> |
| ; CHECK-NEXT: br label %[[INNER_BODY1:.*]] |
| ; CHECK: [[INNER_BODY1]]: |
| ; CHECK-NEXT: [[J2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[INNER_BODY1]] ] |
| ; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i64> [[TMP2]], [[J2]] |
| ; CHECK-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds i8, ptr [[A]], <4 x i64> [[TMP4]] |
| ; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP3]], <4 x ptr> align 1 [[WIDE_GEP3]], <4 x i1> splat (i1 true)) |
| ; CHECK-NEXT: [[TMP5]] = add nuw nsw <4 x i64> [[J2]], splat (i64 1) |
| ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0 |
| ; CHECK-NEXT: br i1 [[TMP7]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]] |
| ; CHECK: [[OUTER_LATCH4]]: |
| ; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i4> [[WIDE_LOAD]], splat (i4 1) |
| ; CHECK-NEXT: call void @llvm.masked.scatter.v4i4.v4p0(<4 x i4> [[TMP8]], <4 x ptr> align 1 [[WIDE_GEP]], <4 x i1> splat (i1 true)) |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) |
| ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; |
| entry: |
| br label %outer.header |
| |
| outer.header: |
| %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ] |
| %vals.ptr = getelementptr inbounds i4, ptr %vals, i64 %i |
| %val = load i4, ptr %vals.ptr, align 1 |
| %i.mul.M = mul nsw i64 %i, %M |
| %val.ext = zext i4 %val to i8 |
| br label %inner.body |
| |
| inner.body: |
| %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ] |
| %idx = add nsw i64 %i.mul.M, %j |
| %A.ptr = getelementptr inbounds i8, ptr %A, i64 %idx |
| store i8 %val.ext, ptr %A.ptr, align 1 |
| %j.next = add nuw nsw i64 %j, 1 |
| %j.cmp = icmp eq i64 %j.next, %M |
| br i1 %j.cmp, label %outer.latch, label %inner.body |
| |
| outer.latch: |
| %val.inc = add i4 %val, 1 |
| store i4 %val.inc, ptr %vals.ptr, align 1 |
| %i.next = add nuw nsw i64 %i, 1 |
| %i.cmp = icmp eq i64 %i.next, %N |
| br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0 |
| |
| exit: |
| ret void |
| } |
| |
| ; --- stride-1 i8 load and store --- |
| ; Same as above with i8, which is not bit-packed in vectors, so a <4 x i8> |
| ; access covers the same bytes as 4 consecutive i8 scalars. |
| define void @stride1_i8_load_store(ptr noalias %A, ptr noalias %vals, i64 %N, i64 %M) { |
| ; CHECK-LABEL: define void @stride1_i8_load_store( |
| ; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[VALS:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH4:.*]] ] |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH4]] ] |
| ; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i8, ptr [[VALS]], <4 x i64> [[VEC_IND]] |
| ; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x ptr> [[WIDE_GEP]], i64 0 |
| ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1 |
| ; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: br label %[[INNER_BODY1:.*]] |
| ; CHECK: [[INNER_BODY1]]: |
| ; CHECK-NEXT: [[J2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_BODY1]] ] |
| ; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i64> [[TMP2]], [[J2]] |
| ; CHECK-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds i8, ptr [[A]], <4 x i64> [[TMP3]] |
| ; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[WIDE_LOAD]], <4 x ptr> align 1 [[WIDE_GEP3]], <4 x i1> splat (i1 true)) |
| ; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[J2]], splat (i64 1) |
| ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0 |
| ; CHECK-NEXT: br i1 [[TMP6]], label %[[OUTER_LATCH4]], label %[[INNER_BODY1]] |
| ; CHECK: [[OUTER_LATCH4]]: |
| ; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i8> [[WIDE_LOAD]], splat (i8 1) |
| ; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x ptr> [[WIDE_GEP]], i64 0 |
| ; CHECK-NEXT: store <4 x i8> [[TMP7]], ptr [[TMP8]], align 1 |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) |
| ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; |
| entry: |
| br label %outer.header |
| |
| outer.header: |
| %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ] |
| %vals.ptr = getelementptr inbounds i8, ptr %vals, i64 %i |
| %val = load i8, ptr %vals.ptr, align 1 |
| %i.mul.M = mul nsw i64 %i, %M |
| br label %inner.body |
| |
| inner.body: |
| %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ] |
| %idx = add nsw i64 %i.mul.M, %j |
| %A.ptr = getelementptr inbounds i8, ptr %A, i64 %idx |
| store i8 %val, ptr %A.ptr, align 1 |
| %j.next = add nuw nsw i64 %j, 1 |
| %j.cmp = icmp eq i64 %j.next, %M |
| br i1 %j.cmp, label %outer.latch, label %inner.body |
| |
| outer.latch: |
| %val.inc = add i8 %val, 1 |
| store i8 %val.inc, ptr %vals.ptr, align 1 |
| %i.next = add nuw nsw i64 %i, 1 |
| %i.cmp = icmp eq i64 %i.next, %N |
| br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0 |
| |
| exit: |
| ret void |
| } |
| |
| !0 = distinct !{!0, !1, !2} |
| !1 = !{!"llvm.loop.vectorize.width", i32 4} |
| !2 = !{!"llvm.loop.vectorize.enable"} |
