| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 6 |
| ; |
| ; RUN: opt -p loop-vectorize -force-vector-width=2 -tail-folding-policy=prefer-fold-tail -S %s | FileCheck %s |
| ; RUN: opt -p loop-vectorize -mattr=+sve -tail-folding-policy=prefer-fold-tail -S %s | FileCheck %s --check-prefix=SVE |
| |
| target triple = "arm64-apple-macosx" |
| |
| define void @load_store_interleave_group(ptr noalias %data, i64 %n) { |
| ; CHECK-LABEL: define void @load_store_interleave_group( |
| ; CHECK-SAME: ptr noalias [[DATA:%.*]], i64 [[N:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: br label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 1 |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] |
| ; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1 |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0 |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ] |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ] |
| ; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]] |
| ; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i1> [[TMP0]], i64 0 |
| ; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]] |
| ; CHECK: [[PRED_STORE_IF]]: |
| ; CHECK-NEXT: [[TMP3:%.*]] = shl nsw i64 [[INDEX]], 1 |
| ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP3]] |
| ; CHECK-NEXT: [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 8 |
| ; CHECK-NEXT: store i64 [[TMP5]], ptr [[TMP4]], align 8 |
| ; CHECK-NEXT: [[TMP6:%.*]] = or disjoint i64 [[TMP3]], 1 |
| ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP6]] |
| ; CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP7]], align 8 |
| ; CHECK-NEXT: store i64 [[TMP8]], ptr [[TMP7]], align 8 |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]] |
| ; CHECK: [[PRED_STORE_CONTINUE]]: |
| ; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP0]], i64 1 |
| ; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]] |
| ; CHECK: [[PRED_STORE_IF1]]: |
| ; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 1 |
| ; CHECK-NEXT: [[TMP11:%.*]] = shl nsw i64 [[TMP10]], 1 |
| ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP11]] |
| ; CHECK-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP12]], align 8 |
| ; CHECK-NEXT: store i64 [[TMP13]], ptr [[TMP12]], align 8 |
| ; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[TMP11]], 1 |
| ; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP14]] |
| ; CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr [[TMP15]], align 8 |
| ; CHECK-NEXT: store i64 [[TMP16]], ptr [[TMP15]], align 8 |
| ; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]] |
| ; CHECK: [[PRED_STORE_CONTINUE2]]: |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2 |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2) |
| ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: br label %[[EXIT:.*]] |
| ; CHECK: [[EXIT]]: |
| ; CHECK-NEXT: ret void |
| ; |
| ; SVE-LABEL: define void @load_store_interleave_group( |
| ; SVE-SAME: ptr noalias [[DATA:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; SVE-NEXT: [[ENTRY:.*:]] |
| ; SVE-NEXT: br label %[[VECTOR_PH:.*]] |
| ; SVE: [[VECTOR_PH]]: |
| ; SVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() |
| ; SVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 |
| ; SVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]]) |
| ; SVE-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; SVE: [[VECTOR_BODY]]: |
| ; SVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; SVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; SVE-NEXT: [[TMP2:%.*]] = shl nsw i64 [[INDEX]], 1 |
| ; SVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP2]] |
| ; SVE-NEXT: [[INTERLEAVED_MASK:%.*]] = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]]) |
| ; SVE-NEXT: [[WIDE_MASKED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr align 8 [[TMP3]], <vscale x 4 x i1> [[INTERLEAVED_MASK]], <vscale x 4 x i64> poison) |
| ; SVE-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_MASKED_VEC]]) |
| ; SVE-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0 |
| ; SVE-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1 |
| ; SVE-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]]) |
| ; SVE-NEXT: [[INTERLEAVED_MASK1:%.*]] = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]]) |
| ; SVE-NEXT: call void @llvm.masked.store.nxv4i64.p0(<vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr align 8 [[TMP3]], <vscale x 4 x i1> [[INTERLEAVED_MASK1]]) |
| ; SVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]] |
| ; SVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) |
| ; SVE-NEXT: [[TMP6:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0 |
| ; SVE-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true |
| ; SVE-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; SVE: [[MIDDLE_BLOCK]]: |
| ; SVE-NEXT: br label %[[EXIT:.*]] |
| ; SVE: [[EXIT]]: |
| ; SVE-NEXT: ret void |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %mul.2 = shl nsw i64 %iv, 1 |
| %data.0 = getelementptr inbounds i64, ptr %data, i64 %mul.2 |
| %l.0 = load i64, ptr %data.0, align 8 |
| store i64 %l.0, ptr %data.0, align 8 |
| %add.1 = or disjoint i64 %mul.2, 1 |
| %data.1 = getelementptr inbounds i64, ptr %data, i64 %add.1 |
| %l.1 = load i64, ptr %data.1, align 8 |
| store i64 %l.1, ptr %data.1, align 8 |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %n |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret void |
| } |
