llvm/test/Transforms/CodeGenPrepare/AArch64/vector-interleave-sink.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=aarch64-linux-gnu -mattr=+sve -codegenprepare %s -S | FileCheck %s

define void @sink_interleave4_to_store(<8 x i16> %src0, <8 x i16> %src1, <8 x i16> %src2, <8 x i16> %src3, ptr %dst, i1 %cond) {
; CHECK-LABEL: define void @sink_interleave4_to_store(
; CHECK-SAME: <8 x i16> [[SRC0:%.*]], <8 x i16> [[SRC1:%.*]], <8 x i16> [[SRC2:%.*]], <8 x i16> [[SRC3:%.*]], ptr [[DST:%.*]], i1 [[COND:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br i1 [[COND]], label %[[STORE:.*]], label %[[EXIT:.*]]
; CHECK:       [[STORE]]:
; CHECK-NEXT:    [[TMP0:%.*]] = tail call <32 x i16> @llvm.vector.interleave4.v32i16(<8 x i16> [[SRC0]], <8 x i16> [[SRC1]], <8 x i16> [[SRC2]], <8 x i16> [[SRC3]])
; CHECK-NEXT:    store <32 x i16> [[TMP0]], ptr [[DST]], align 64
; CHECK-NEXT:    br label %[[EXIT]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  %interleave = tail call <32 x i16> @llvm.vector.interleave4(
  <8 x i16> %src0, <8 x i16> %src1,
  <8 x i16> %src2, <8 x i16> %src3)
  br i1 %cond, label %store, label %exit

store:
  store <32 x i16> %interleave, ptr %dst
  br label %exit

exit:
  ret void
}

define void @sink_interleave4_to_store_scalable(<vscale x 8 x i16> %src0, <vscale x 8 x i16> %src1, <vscale x 8 x i16> %src2, <vscale x 8 x i16> %src3, ptr %dst, i1 %cond) {
; CHECK-LABEL: define void @sink_interleave4_to_store_scalable(
; CHECK-SAME: <vscale x 8 x i16> [[SRC0:%.*]], <vscale x 8 x i16> [[SRC1:%.*]], <vscale x 8 x i16> [[SRC2:%.*]], <vscale x 8 x i16> [[SRC3:%.*]], ptr [[DST:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br i1 [[COND]], label %[[STORE:.*]], label %[[EXIT:.*]]
; CHECK:       [[STORE]]:
; CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.interleave4.nxv32i16(<vscale x 8 x i16> [[SRC0]], <vscale x 8 x i16> [[SRC1]], <vscale x 8 x i16> [[SRC2]], <vscale x 8 x i16> [[SRC3]])
; CHECK-NEXT:    store <vscale x 32 x i16> [[TMP0]], ptr [[DST]], align 64
; CHECK-NEXT:    br label %[[EXIT]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  %interleave = tail call <vscale x 32 x i16> @llvm.vector.interleave4(
  <vscale x 8 x i16> %src0, <vscale x 8 x i16> %src1,
  <vscale x 8 x i16> %src2, <vscale x 8 x i16> %src3)
  br i1 %cond, label %store, label %exit

store:
  store <vscale x 32 x i16> %interleave, ptr %dst
  br label %exit

exit:
  ret void
}

define void @sink_interleave6_to_store(<8 x i16> %src0, <8 x i16> %src1, <8 x i16> %src2, <8 x i16> %src3, <8 x i16> %src4, <8 x i16> %src5, ptr %dst, i1 %cond) {
; CHECK-LABEL: define void @sink_interleave6_to_store(
; CHECK-SAME: <8 x i16> [[SRC0:%.*]], <8 x i16> [[SRC1:%.*]], <8 x i16> [[SRC2:%.*]], <8 x i16> [[SRC3:%.*]], <8 x i16> [[SRC4:%.*]], <8 x i16> [[SRC5:%.*]], ptr [[DST:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br i1 [[COND]], label %[[STORE:.*]], label %[[EXIT:.*]]
; CHECK:       [[STORE]]:
; CHECK-NEXT:    [[TMP0:%.*]] = tail call <48 x i16> @llvm.vector.interleave6.v48i16(<8 x i16> [[SRC0]], <8 x i16> [[SRC1]], <8 x i16> [[SRC2]], <8 x i16> [[SRC3]], <8 x i16> [[SRC4]], <8 x i16> [[SRC5]])
; CHECK-NEXT:    store <48 x i16> [[TMP0]], ptr [[DST]], align 128
; CHECK-NEXT:    br label %[[EXIT]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  %interleave = tail call <48 x i16> @llvm.vector.interleave6(
  <8 x i16> %src0, <8 x i16> %src1, <8 x i16> %src2,
  <8 x i16> %src3, <8 x i16> %src4, <8 x i16> %src5)
  br i1 %cond, label %store, label %exit

store:
  store <48 x i16> %interleave, ptr %dst
  br label %exit

exit:
  ret void
}

define void @do_not_sink_interleave6_to_loop(<8 x i16> %src0, <8 x i16> %src1, <8 x i16> %src2, <8 x i16> %src3, <8 x i16> %src4, <8 x i16> %src5, ptr %dst, i64 %n) {
; CHECK-LABEL: define void @do_not_sink_interleave6_to_loop(
; CHECK-SAME: <8 x i16> [[SRC0:%.*]], <8 x i16> [[SRC1:%.*]], <8 x i16> [[SRC2:%.*]], <8 x i16> [[SRC3:%.*]], <8 x i16> [[SRC4:%.*]], <8 x i16> [[SRC5:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    [[INTERLEAVE:%.*]] = tail call <48 x i16> @llvm.vector.interleave6.v48i16(<8 x i16> [[SRC0]], <8 x i16> [[SRC1]], <8 x i16> [[SRC2]], <8 x i16> [[SRC3]], <8 x i16> [[SRC4]], <8 x i16> [[SRC5]])
; CHECK-NEXT:    br label %[[LOOP:.*]]
; CHECK:       [[LOOP]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT:    [[PTR:%.*]] = getelementptr <48 x i16>, ptr [[DST]], i64 [[IV]]
; CHECK-NEXT:    store <48 x i16> [[INTERLEAVE]], ptr [[PTR]], align 128
; CHECK-NEXT:    [[NEXT]] = add nuw i64 [[IV]], 1
; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  %interleave = tail call <48 x i16> @llvm.vector.interleave6(
  <8 x i16> %src0, <8 x i16> %src1, <8 x i16> %src2,
  <8 x i16> %src3, <8 x i16> %src4, <8 x i16> %src5)
  br label %loop

loop:
  %iv = phi i64 [ 0, %entry ], [ %next, %loop ]
  %ptr = getelementptr <48 x i16>, ptr %dst, i64 %iv
  store <48 x i16> %interleave, ptr %ptr
  %next = add nuw i64 %iv, 1
  %done = icmp eq i64 %next, %n
  br i1 %done, label %exit, label %loop

exit:
  ret void
}