llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=0 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI

define <4 x half> @interleave2_v4f16(<2 x half> %vec0, <2 x half> %vec1) {
; CHECK-LABEL: interleave2_v4f16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v0.4h, v0.4h, v1.4h
; CHECK-NEXT:    ret
  %retval = call <4 x half> @llvm.vector.interleave2.v4f16(<2 x half> %vec0, <2 x half> %vec1)
  ret <4 x half> %retval
}

define <8 x half> @interleave2_v8f16(<4 x half> %vec0, <4 x half> %vec1) {
; CHECK-SD-LABEL: interleave2_v8f16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-SD-NEXT:    adrp x8, .LCPI1_0
; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI1_0]
; CHECK-SD-NEXT:    tbl v0.16b, { v0.16b }, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: interleave2_v8f16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    zip1 v0.8h, v0.8h, v1.8h
; CHECK-GI-NEXT:    ret
  %retval = call <8 x half> @llvm.vector.interleave2.v8f16(<4 x half> %vec0, <4 x half> %vec1)
  ret <8 x half> %retval
}

define <16 x half> @interleave2_v16f16(<8 x half> %vec0, <8 x half> %vec1) {
; CHECK-LABEL: interleave2_v16f16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v2.8h, v0.8h, v1.8h
; CHECK-NEXT:    zip2 v1.8h, v0.8h, v1.8h
; CHECK-NEXT:    mov v0.16b, v2.16b
; CHECK-NEXT:    ret
  %retval = call <16 x half> @llvm.vector.interleave2.v16f16(<8 x half> %vec0, <8 x half> %vec1)
  ret <16 x half> %retval
}

define <4 x float> @interleave2_v4f32(<2 x float> %vec0, <2 x float> %vec1) {
; CHECK-SD-LABEL: interleave2_v4f32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-SD-NEXT:    rev64 v1.4s, v0.4s
; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: interleave2_v4f32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    zip1 v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    ret
  %retval = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %vec0, <2 x float> %vec1)
  ret <4 x float> %retval
}

define <8 x float> @interleave2_v8f32(<4 x float> %vec0, <4 x float> %vec1) {
; CHECK-LABEL: interleave2_v8f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v2.4s, v0.4s, v1.4s
; CHECK-NEXT:    zip2 v1.4s, v0.4s, v1.4s
; CHECK-NEXT:    mov v0.16b, v2.16b
; CHECK-NEXT:    ret
  %retval = call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> %vec0, <4 x float> %vec1)
  ret <8 x float> %retval
}

define <4 x double> @interleave2_v4f64(<2 x double> %vec0, <2 x double> %vec1) {
; CHECK-LABEL: interleave2_v4f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v2.2d, v0.2d, v1.2d
; CHECK-NEXT:    zip2 v1.2d, v0.2d, v1.2d
; CHECK-NEXT:    mov v0.16b, v2.16b
; CHECK-NEXT:    ret
  %retval = call <4 x double>@llvm.vector.interleave2.v4f64(<2 x double> %vec0, <2 x double> %vec1)
  ret <4 x double> %retval
}

; Integers

define <32 x i8> @interleave2_v32i8(<16 x i8> %vec0, <16 x i8> %vec1) {
; CHECK-LABEL: interleave2_v32i8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v2.16b, v0.16b, v1.16b
; CHECK-NEXT:    zip2 v1.16b, v0.16b, v1.16b
; CHECK-NEXT:    mov v0.16b, v2.16b
; CHECK-NEXT:    ret
 %retval = call <32 x i8> @llvm.vector.interleave2.v32i8(<16 x i8> %vec0, <16 x i8> %vec1)
  ret <32 x i8> %retval
}

define <16 x i16> @interleave2_v16i16(<8 x i16> %vec0, <8 x i16> %vec1) {
; CHECK-LABEL: interleave2_v16i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v2.8h, v0.8h, v1.8h
; CHECK-NEXT:    zip2 v1.8h, v0.8h, v1.8h
; CHECK-NEXT:    mov v0.16b, v2.16b
; CHECK-NEXT:    ret
  %retval = call <16 x i16> @llvm.vector.interleave2.v16i16(<8 x i16> %vec0, <8 x i16> %vec1)
  ret <16 x i16> %retval
}

define <8 x i32> @interleave2_v8i32(<4 x i32> %vec0, <4 x i32> %vec1) {
; CHECK-LABEL: interleave2_v8i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v2.4s, v0.4s, v1.4s
; CHECK-NEXT:    zip2 v1.4s, v0.4s, v1.4s
; CHECK-NEXT:    mov v0.16b, v2.16b
; CHECK-NEXT:    ret
  %retval = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %vec0, <4 x i32> %vec1)
  ret <8 x i32> %retval
}

define <4 x i64> @interleave2_v4i64(<2 x i64> %vec0, <2 x i64> %vec1) {
; CHECK-LABEL: interleave2_v4i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v2.2d, v0.2d, v1.2d
; CHECK-NEXT:    zip2 v1.2d, v0.2d, v1.2d
; CHECK-NEXT:    mov v0.16b, v2.16b
; CHECK-NEXT:    ret
  %retval = call <4 x i64> @llvm.vector.interleave2.v4i64(<2 x i64> %vec0, <2 x i64> %vec1)
  ret <4 x i64> %retval
}

define <4 x i16> @interleave2_same_const_splat_v4i16() {
; CHECK-SD-LABEL: interleave2_same_const_splat_v4i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    movi v0.4h, #3
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: interleave2_same_const_splat_v4i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    mov w8, #3 // =0x3
; CHECK-GI-NEXT:    fmov s0, w8
; CHECK-GI-NEXT:    mov v0.h[1], w8
; CHECK-GI-NEXT:    zip1 v0.4h, v0.4h, v0.4h
; CHECK-GI-NEXT:    ret
  %retval = call <4 x i16> @llvm.vector.interleave2.v4i16(<2 x i16> splat(i16 3), <2 x i16> splat(i16 3))
  ret <4 x i16> %retval
}

define <4 x i16> @interleave2_diff_const_splat_v4i16() {
; CHECK-SD-LABEL: interleave2_diff_const_splat_v4i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov x8, #1125899907104768 // =0x4000000040000
; CHECK-SD-NEXT:    orr x8, x8, #0x300000003
; CHECK-SD-NEXT:    fmov d0, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: interleave2_diff_const_splat_v4i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    mov w8, #3 // =0x3
; CHECK-GI-NEXT:    mov w9, #4 // =0x4
; CHECK-GI-NEXT:    fmov s0, w8
; CHECK-GI-NEXT:    fmov s1, w9
; CHECK-GI-NEXT:    mov v0.h[1], w8
; CHECK-GI-NEXT:    mov v1.h[1], w9
; CHECK-GI-NEXT:    zip1 v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    ret
  %retval = call <4 x i16> @llvm.vector.interleave2.v4i16(<2 x i16> splat(i16 3), <2 x i16> splat(i16 4))
  ret <4 x i16> %retval
}

define <4 x i16> @interleave2_same_nonconst_splat_v4i16(i16 %a) {
; CHECK-SD-LABEL: interleave2_same_nonconst_splat_v4i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    dup v0.4h, w0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: interleave2_same_nonconst_splat_v4i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    dup v0.4h, w0
; CHECK-GI-NEXT:    zip1 v0.4h, v0.4h, v0.4h
; CHECK-GI-NEXT:    ret
  %ins = insertelement <2 x i16> poison, i16 %a, i32 0
  %splat = shufflevector <2 x i16> %ins, <2 x i16> poison, <2 x i32> <i32 0, i32 0>
  %retval = call <4 x i16> @llvm.vector.interleave2.v4i16(<2 x i16> %splat, <2 x i16> %splat)
  ret <4 x i16> %retval
}

define <4 x i16> @interleave2_diff_nonconst_splat_v4i16(i16 %a, i16 %b) {
; CHECK-SD-LABEL: interleave2_diff_nonconst_splat_v4i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    fmov s0, w0
; CHECK-SD-NEXT:    mov v0.h[1], w0
; CHECK-SD-NEXT:    mov v0.h[2], w1
; CHECK-SD-NEXT:    mov v0.h[3], w1
; CHECK-SD-NEXT:    rev32 v1.4h, v0.4h
; CHECK-SD-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: interleave2_diff_nonconst_splat_v4i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    dup v0.4h, w0
; CHECK-GI-NEXT:    dup v1.4h, w1
; CHECK-GI-NEXT:    zip1 v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    ret
  %ins1 = insertelement <2 x i16> poison, i16 %a, i32 0
  %splat1 = shufflevector <2 x i16> %ins1, <2 x i16> poison, <2 x i32> <i32 0, i32 0>
  %ins2 = insertelement <2 x i16> poison, i16 %b, i32 0
  %splat2 = shufflevector <2 x i16> %ins2, <2 x i16> poison, <2 x i32> <i32 0, i32 0>
  %retval = call <4 x i16> @llvm.vector.interleave2.v4i16(<2 x i16> %splat1, <2 x i16> %splat2)
  ret <4 x i16> %retval
}

define <32 x i8> @interleave4_v32i8(<8 x i8> %vec0, <8 x i8> %vec1, <8 x i8> %vec2, <8 x i8> %vec3) {
; CHECK-LABEL: interleave4_v32i8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.8b, v1.8b, v3.8b
; CHECK-NEXT:    zip1 v5.8b, v0.8b, v2.8b
; CHECK-NEXT:    zip2 v1.8b, v1.8b, v3.8b
; CHECK-NEXT:    zip2 v2.8b, v0.8b, v2.8b
; CHECK-NEXT:    zip1 v0.16b, v5.16b, v4.16b
; CHECK-NEXT:    zip1 v1.16b, v2.16b, v1.16b
; CHECK-NEXT:    ret
  %retval = call <32 x i8> @llvm.vector.interleave4.v32i8(<8 x i8> %vec0, <8 x i8> %vec1, <8 x i8> %vec2, <8 x i8> %vec3)
  ret <32 x i8> %retval
}

define <64 x i8> @interleave4_v64i8(<16 x i8> %vec0, <16 x i8> %vec1, <16 x i8> %vec2, <16 x i8> %vec3) {
; CHECK-LABEL: interleave4_v64i8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.16b, v1.16b, v3.16b
; CHECK-NEXT:    zip1 v5.16b, v0.16b, v2.16b
; CHECK-NEXT:    zip2 v3.16b, v1.16b, v3.16b
; CHECK-NEXT:    zip2 v6.16b, v0.16b, v2.16b
; CHECK-NEXT:    zip1 v0.16b, v5.16b, v4.16b
; CHECK-NEXT:    zip2 v1.16b, v5.16b, v4.16b
; CHECK-NEXT:    zip1 v2.16b, v6.16b, v3.16b
; CHECK-NEXT:    zip2 v3.16b, v6.16b, v3.16b
; CHECK-NEXT:    ret
  %retval = call <64 x i8> @llvm.vector.interleave4.v64i8(<16 x i8> %vec0, <16 x i8> %vec1, <16 x i8> %vec2, <16 x i8> %vec3)
  ret <64 x i8> %retval
}

define <16 x i16> @interleave4_v16i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2, <4 x i16> %vec3) {
; CHECK-LABEL: interleave4_v16i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.4h, v1.4h, v3.4h
; CHECK-NEXT:    zip1 v5.4h, v0.4h, v2.4h
; CHECK-NEXT:    zip2 v1.4h, v1.4h, v3.4h
; CHECK-NEXT:    zip2 v2.4h, v0.4h, v2.4h
; CHECK-NEXT:    zip1 v0.8h, v5.8h, v4.8h
; CHECK-NEXT:    zip1 v1.8h, v2.8h, v1.8h
; CHECK-NEXT:    ret
  %retval = call <16 x i16> @llvm.vector.interleave4.v16i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2, <4 x i16> %vec3)
  ret <16 x i16> %retval
}

define <32 x i16> @interleave4_v32i16(<8 x i16> %vec0, <8 x i16> %vec1, <8 x i16> %vec2, <8 x i16> %vec3) {
; CHECK-LABEL: interleave4_v32i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.8h, v1.8h, v3.8h
; CHECK-NEXT:    zip1 v5.8h, v0.8h, v2.8h
; CHECK-NEXT:    zip2 v3.8h, v1.8h, v3.8h
; CHECK-NEXT:    zip2 v6.8h, v0.8h, v2.8h
; CHECK-NEXT:    zip1 v0.8h, v5.8h, v4.8h
; CHECK-NEXT:    zip2 v1.8h, v5.8h, v4.8h
; CHECK-NEXT:    zip1 v2.8h, v6.8h, v3.8h
; CHECK-NEXT:    zip2 v3.8h, v6.8h, v3.8h
; CHECK-NEXT:    ret
  %retval = call <32 x i16> @llvm.vector.interleave4.v32i16(<8 x i16> %vec0, <8 x i16> %vec1, <8 x i16> %vec2, <8 x i16> %vec3)
  ret <32 x i16> %retval
}

define <8 x i32> @interleave4_v8i32(<2 x i32> %vec0, <2 x i32> %vec1, <2 x i32> %vec2, <2 x i32> %vec3) {
; CHECK-LABEL: interleave4_v8i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.2s, v1.2s, v3.2s
; CHECK-NEXT:    zip1 v5.2s, v0.2s, v2.2s
; CHECK-NEXT:    zip2 v1.2s, v1.2s, v3.2s
; CHECK-NEXT:    zip2 v2.2s, v0.2s, v2.2s
; CHECK-NEXT:    zip1 v0.4s, v5.4s, v4.4s
; CHECK-NEXT:    zip1 v1.4s, v2.4s, v1.4s
; CHECK-NEXT:    ret
  %retval = call <8 x i32> @llvm.vector.interleave4.v8i32(<2 x i32> %vec0, <2 x i32> %vec1, <2 x i32> %vec2, <2 x i32> %vec3)
  ret <8 x i32> %retval
}

define <16 x i32> @interleave4_v16i32(<4 x i32> %vec0, <4 x i32> %vec1, <4 x i32> %vec2, <4 x i32> %vec3) {
; CHECK-LABEL: interleave4_v16i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.4s, v1.4s, v3.4s
; CHECK-NEXT:    zip1 v5.4s, v0.4s, v2.4s
; CHECK-NEXT:    zip2 v3.4s, v1.4s, v3.4s
; CHECK-NEXT:    zip2 v6.4s, v0.4s, v2.4s
; CHECK-NEXT:    zip1 v0.4s, v5.4s, v4.4s
; CHECK-NEXT:    zip2 v1.4s, v5.4s, v4.4s
; CHECK-NEXT:    zip1 v2.4s, v6.4s, v3.4s
; CHECK-NEXT:    zip2 v3.4s, v6.4s, v3.4s
; CHECK-NEXT:    ret
  %retval = call <16 x i32> @llvm.vector.interleave4.v16i32(<4 x i32> %vec0, <4 x i32> %vec1, <4 x i32> %vec2, <4 x i32> %vec3)
  ret <16 x i32> %retval
}

define <4 x i64> @interleave4_v4i64(<1 x i64> %vec0, <1 x i64> %vec1, <1 x i64> %vec2, <1 x i64> %vec3) {
; CHECK-LABEL: interleave4_v4i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-NEXT:    mov v2.d[1], v3.d[0]
; CHECK-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-NEXT:    mov v1.16b, v2.16b
; CHECK-NEXT:    ret
  %retval = call <4 x i64> @llvm.vector.interleave4.v4i64(<1 x i64> %vec0, <1 x i64> %vec1, <1 x i64> %vec2, <1 x i64> %vec3)
  ret <4 x i64> %retval
}

define <8 x i64> @interleave4_v8i64(<2 x i64> %vec0, <2 x i64> %vec1, <2 x i64> %vec2, <2 x i64> %vec3) {
; CHECK-LABEL: interleave4_v8i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.2d, v1.2d, v3.2d
; CHECK-NEXT:    zip1 v5.2d, v0.2d, v2.2d
; CHECK-NEXT:    zip2 v3.2d, v1.2d, v3.2d
; CHECK-NEXT:    zip2 v6.2d, v0.2d, v2.2d
; CHECK-NEXT:    zip1 v0.2d, v5.2d, v4.2d
; CHECK-NEXT:    zip2 v1.2d, v5.2d, v4.2d
; CHECK-NEXT:    zip1 v2.2d, v6.2d, v3.2d
; CHECK-NEXT:    zip2 v3.2d, v6.2d, v3.2d
; CHECK-NEXT:    ret
  %retval = call <8 x i64> @llvm.vector.interleave4.v8i64(<2 x i64> %vec0, <2 x i64> %vec1, <2 x i64> %vec2, <2 x i64> %vec3)
  ret <8 x i64> %retval
}

define <16 x half> @interleave4_v16f16(<4 x half> %vec0, <4 x half> %vec1, <4 x half> %vec2, <4 x half> %vec3) {
; CHECK-LABEL: interleave4_v16f16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.4h, v1.4h, v3.4h
; CHECK-NEXT:    zip1 v5.4h, v0.4h, v2.4h
; CHECK-NEXT:    zip2 v1.4h, v1.4h, v3.4h
; CHECK-NEXT:    zip2 v2.4h, v0.4h, v2.4h
; CHECK-NEXT:    zip1 v0.8h, v5.8h, v4.8h
; CHECK-NEXT:    zip1 v1.8h, v2.8h, v1.8h
; CHECK-NEXT:    ret
  %retval = call <16 x half> @llvm.vector.interleave4.v16f16(<4 x half> %vec0, <4 x half> %vec1, <4 x half> %vec2, <4 x half> %vec3)
  ret <16 x half> %retval
}

define <32 x half> @interleave4_v32f16(<8 x half> %vec0, <8 x half> %vec1, <8 x half> %vec2, <8 x half> %vec3) {
; CHECK-LABEL: interleave4_v32f16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.8h, v1.8h, v3.8h
; CHECK-NEXT:    zip1 v5.8h, v0.8h, v2.8h
; CHECK-NEXT:    zip2 v3.8h, v1.8h, v3.8h
; CHECK-NEXT:    zip2 v6.8h, v0.8h, v2.8h
; CHECK-NEXT:    zip1 v0.8h, v5.8h, v4.8h
; CHECK-NEXT:    zip2 v1.8h, v5.8h, v4.8h
; CHECK-NEXT:    zip1 v2.8h, v6.8h, v3.8h
; CHECK-NEXT:    zip2 v3.8h, v6.8h, v3.8h
; CHECK-NEXT:    ret
  %retval = call <32 x half> @llvm.vector.interleave4.v32f16(<8 x half> %vec0, <8 x half> %vec1, <8 x half> %vec2, <8 x half> %vec3)
  ret <32 x half> %retval
}

define <8 x float> @interleave4_v8f32(<2 x float> %vec0, <2 x float> %vec1, <2 x float> %vec2, <2 x float> %vec3) {
; CHECK-LABEL: interleave4_v8f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.2s, v1.2s, v3.2s
; CHECK-NEXT:    zip1 v5.2s, v0.2s, v2.2s
; CHECK-NEXT:    zip2 v1.2s, v1.2s, v3.2s
; CHECK-NEXT:    zip2 v2.2s, v0.2s, v2.2s
; CHECK-NEXT:    zip1 v0.4s, v5.4s, v4.4s
; CHECK-NEXT:    zip1 v1.4s, v2.4s, v1.4s
; CHECK-NEXT:    ret
  %retval = call <8 x float> @llvm.vector.interleave4.v8f32(<2 x float> %vec0, <2 x float> %vec1, <2 x float> %vec2, <2 x float> %vec3)
  ret <8 x float> %retval
}

define <16 x float> @interleave4_v16f32(<4 x float> %vec0, <4 x float> %vec1, <4 x float> %vec2, <4 x float> %vec3) {
; CHECK-LABEL: interleave4_v16f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.4s, v1.4s, v3.4s
; CHECK-NEXT:    zip1 v5.4s, v0.4s, v2.4s
; CHECK-NEXT:    zip2 v3.4s, v1.4s, v3.4s
; CHECK-NEXT:    zip2 v6.4s, v0.4s, v2.4s
; CHECK-NEXT:    zip1 v0.4s, v5.4s, v4.4s
; CHECK-NEXT:    zip2 v1.4s, v5.4s, v4.4s
; CHECK-NEXT:    zip1 v2.4s, v6.4s, v3.4s
; CHECK-NEXT:    zip2 v3.4s, v6.4s, v3.4s
; CHECK-NEXT:    ret
  %retval = call <16 x float> @llvm.vector.interleave4.v16f32(<4 x float> %vec0, <4 x float> %vec1, <4 x float> %vec2, <4 x float> %vec3)
  ret <16 x float> %retval
}

define <4 x double> @interleave4_v4f64(<1 x double> %vec0, <1 x double> %vec1, <1 x double> %vec2, <1 x double> %vec3) {
; CHECK-LABEL: interleave4_v4f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-NEXT:    mov v2.d[1], v3.d[0]
; CHECK-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-NEXT:    mov v1.16b, v2.16b
; CHECK-NEXT:    ret
  %retval = call <4 x double> @llvm.vector.interleave4.v4f64(<1 x double> %vec0, <1 x double> %vec1, <1 x double> %vec2, <1 x double> %vec3)
  ret <4 x double> %retval
}

define <8 x double> @interleave4_v8f64(<2 x double> %vec0, <2 x double> %vec1, <2 x double> %vec2, <2 x double> %vec3) {
; CHECK-LABEL: interleave4_v8f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.2d, v1.2d, v3.2d
; CHECK-NEXT:    zip1 v5.2d, v0.2d, v2.2d
; CHECK-NEXT:    zip2 v3.2d, v1.2d, v3.2d
; CHECK-NEXT:    zip2 v6.2d, v0.2d, v2.2d
; CHECK-NEXT:    zip1 v0.2d, v5.2d, v4.2d
; CHECK-NEXT:    zip2 v1.2d, v5.2d, v4.2d
; CHECK-NEXT:    zip1 v2.2d, v6.2d, v3.2d
; CHECK-NEXT:    zip2 v3.2d, v6.2d, v3.2d
; CHECK-NEXT:    ret
  %retval = call <8 x double> @llvm.vector.interleave4.v8f64(<2 x double> %vec0, <2 x double> %vec1, <2 x double> %vec2, <2 x double> %vec3)
  ret <8 x double> %retval
}

define <16 x bfloat> @interleave4_v16bf16(<4 x bfloat> %vec0, <4 x bfloat> %vec1, <4 x bfloat> %vec2, <4 x bfloat> %vec3) {
; CHECK-LABEL: interleave4_v16bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.4h, v1.4h, v3.4h
; CHECK-NEXT:    zip1 v5.4h, v0.4h, v2.4h
; CHECK-NEXT:    zip2 v1.4h, v1.4h, v3.4h
; CHECK-NEXT:    zip2 v2.4h, v0.4h, v2.4h
; CHECK-NEXT:    zip1 v0.8h, v5.8h, v4.8h
; CHECK-NEXT:    zip1 v1.8h, v2.8h, v1.8h
; CHECK-NEXT:    ret
  %retval = call <16 x bfloat> @llvm.vector.interleave4.v16bf16(<4 x bfloat> %vec0, <4 x bfloat> %vec1, <4 x bfloat> %vec2, <4 x bfloat> %vec3)
  ret <16 x bfloat> %retval
}

define <32 x bfloat> @interleave4_v32bf16(<8 x bfloat> %vec0, <8 x bfloat> %vec1, <8 x bfloat> %vec2, <8 x bfloat> %vec3) {
; CHECK-LABEL: interleave4_v32bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    zip1 v4.8h, v1.8h, v3.8h
; CHECK-NEXT:    zip1 v5.8h, v0.8h, v2.8h
; CHECK-NEXT:    zip2 v3.8h, v1.8h, v3.8h
; CHECK-NEXT:    zip2 v6.8h, v0.8h, v2.8h
; CHECK-NEXT:    zip1 v0.8h, v5.8h, v4.8h
; CHECK-NEXT:    zip2 v1.8h, v5.8h, v4.8h
; CHECK-NEXT:    zip1 v2.8h, v6.8h, v3.8h
; CHECK-NEXT:    zip2 v3.8h, v6.8h, v3.8h
; CHECK-NEXT:    ret
  %retval = call <32 x bfloat> @llvm.vector.interleave4.v32bf16(<8 x bfloat> %vec0, <8 x bfloat> %vec1, <8 x bfloat> %vec2, <8 x bfloat> %vec3)
  ret <32 x bfloat> %retval
}

define <6 x double> @interleave3_v6f64(ptr %p, ptr %p0, <2 x double> %vec0, <2 x double> %vec1, <2 x double> %vec2) {
; CHECK-LABEL: interleave3_v6f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #48
; CHECK-NEXT:    .cfi_def_cfa_offset 48
; CHECK-NEXT:    // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    mov x8, sp
; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    st3 { v0.2d, v1.2d, v2.2d }, [x8]
; CHECK-NEXT:    ldp q0, q2, [sp]
; CHECK-NEXT:    ldr q4, [sp, #32]
; CHECK-NEXT:    ext v5.16b, v4.16b, v4.16b, #8
; CHECK-NEXT:    // kill: def $d4 killed $d4 killed $q4
; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
; CHECK-NEXT:    ext v3.16b, v2.16b, v2.16b, #8
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $q2
; CHECK-NEXT:    // kill: def $d5 killed $d5 killed $q5
; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
; CHECK-NEXT:    // kill: def $d3 killed $d3 killed $q3
; CHECK-NEXT:    add sp, sp, #48
; CHECK-NEXT:    ret
  %retval = call <6 x double> @llvm.vector.interleave3.v6f64(<2 x double> %vec0, <2 x double> %vec1, <2 x double> %vec2)
  ret <6 x double> %retval
}

define <12 x float> @interleave3_v12f32(ptr %p, ptr %p0, <4 x float> %vec0, <4 x float> %vec1, <4 x float> %vec2) {
; CHECK-LABEL: interleave3_v12f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #48
; CHECK-NEXT:    .cfi_def_cfa_offset 48
; CHECK-NEXT:    // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    mov x9, sp
; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    st3 { v0.4s, v1.4s, v2.4s }, [x9]
; CHECK-NEXT:    ldp q1, q0, [sp, #16]
; CHECK-NEXT:    ldr q2, [sp]
; CHECK-NEXT:    stp q1, q0, [x8, #16]
; CHECK-NEXT:    str q2, [x8]
; CHECK-NEXT:    add sp, sp, #48
; CHECK-NEXT:    ret
  %retval = call <12 x float> @llvm.vector.interleave3.v12f32(<4 x float> %vec0, <4 x float> %vec1, <4 x float> %vec2)
  ret <12 x float> %retval
}

define <24 x i16> @interleave3_v24i16(ptr %p, ptr %p0, <8 x i16> %vec0, <8 x i16> %vec1, <8 x i16> %vec2) {
; CHECK-LABEL: interleave3_v24i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #48
; CHECK-NEXT:    .cfi_def_cfa_offset 48
; CHECK-NEXT:    // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    mov x9, sp
; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    st3 { v0.8h, v1.8h, v2.8h }, [x9]
; CHECK-NEXT:    ldp q1, q0, [sp, #16]
; CHECK-NEXT:    ldr q2, [sp]
; CHECK-NEXT:    stp q1, q0, [x8, #16]
; CHECK-NEXT:    str q2, [x8]
; CHECK-NEXT:    add sp, sp, #48
; CHECK-NEXT:    ret
  %retval = call <24 x i16> @llvm.vector.interleave3.v24i16(<8 x i16> %vec0, <8 x i16> %vec1, <8 x i16> %vec2)
  ret <24 x i16> %retval
}

define <24 x half> @interleave3_v24f16(ptr %p, ptr %p0, <8 x half> %vec0, <8 x half> %vec1, <8 x half> %vec2) {
; CHECK-LABEL: interleave3_v24f16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #48
; CHECK-NEXT:    .cfi_def_cfa_offset 48
; CHECK-NEXT:    // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    mov x9, sp
; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    st3 { v0.8h, v1.8h, v2.8h }, [x9]
; CHECK-NEXT:    ldp q1, q0, [sp, #16]
; CHECK-NEXT:    ldr q2, [sp]
; CHECK-NEXT:    stp q1, q0, [x8, #16]
; CHECK-NEXT:    str q2, [x8]
; CHECK-NEXT:    add sp, sp, #48
; CHECK-NEXT:    ret
  %retval = call <24 x half> @llvm.vector.interleave3.v24f16(<8 x half> %vec0, <8 x half> %vec1, <8 x half> %vec2)
  ret <24 x half> %retval
}

define <24 x bfloat> @interleave3_v24bf16(ptr %p, ptr %p0, <8 x bfloat> %vec0, <8 x bfloat> %vec1, <8 x bfloat> %vec2) {
; CHECK-LABEL: interleave3_v24bf16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #48
; CHECK-NEXT:    .cfi_def_cfa_offset 48
; CHECK-NEXT:    // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    mov x9, sp
; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    st3 { v0.8h, v1.8h, v2.8h }, [x9]
; CHECK-NEXT:    ldp q1, q0, [sp, #16]
; CHECK-NEXT:    ldr q2, [sp]
; CHECK-NEXT:    stp q1, q0, [x8, #16]
; CHECK-NEXT:    str q2, [x8]
; CHECK-NEXT:    add sp, sp, #48
; CHECK-NEXT:    ret
  %retval = call <24 x bfloat> @llvm.vector.interleave3.v24bf16(<8 x bfloat> %vec0, <8 x bfloat> %vec1, <8 x bfloat> %vec2)
  ret <24 x bfloat> %retval
}

define <48 x i8> @interleave3_v48i8(ptr %p, ptr %p0, <16 x i8> %vec0, <16 x i8> %vec1, <16 x i8> %vec2) {
; CHECK-LABEL: interleave3_v48i8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #48
; CHECK-NEXT:    .cfi_def_cfa_offset 48
; CHECK-NEXT:    // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    mov x9, sp
; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT:    st3 { v0.16b, v1.16b, v2.16b }, [x9]
; CHECK-NEXT:    ldp q1, q0, [sp, #16]
; CHECK-NEXT:    ldr q2, [sp]
; CHECK-NEXT:    stp q1, q0, [x8, #16]
; CHECK-NEXT:    str q2, [x8]
; CHECK-NEXT:    add sp, sp, #48
; CHECK-NEXT:    ret
  %retval = call <48 x i8> @llvm.vector.interleave3.v48i8(<16 x i8> %vec0, <16 x i8> %vec1, <16 x i8> %vec2)
  ret <48 x i8> %retval
}

define <12 x i16> @interleave3_v12i16(ptr %p, ptr %p0, <4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2) {
; CHECK-LABEL: interleave3_v12i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #32
; CHECK-NEXT:    .cfi_def_cfa_offset 32
; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
; CHECK-NEXT:    add x9, sp, #8
; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
; CHECK-NEXT:    st3 { v0.4h, v1.4h, v2.4h }, [x9]
; CHECK-NEXT:    ldp d1, d0, [sp, #8]
; CHECK-NEXT:    mov v1.d[1], v0.d[0]
; CHECK-NEXT:    ldr d0, [sp, #24]
; CHECK-NEXT:    str d0, [x8, #16]
; CHECK-NEXT:    str q1, [x8]
; CHECK-NEXT:    add sp, sp, #32
; CHECK-NEXT:    ret
  %retval = call <12 x i16> @llvm.vector.interleave3.v12i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2)
  ret <12 x i16> %retval
}

define <24 x i8> @interleave3_v24i8(ptr %p, ptr %p0, <8 x i8> %vec0, <8 x i8> %vec1, <8 x i8> %vec2) {
; CHECK-LABEL: interleave3_v24i8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #32
; CHECK-NEXT:    .cfi_def_cfa_offset 32
; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
; CHECK-NEXT:    add x9, sp, #8
; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
; CHECK-NEXT:    st3 { v0.8b, v1.8b, v2.8b }, [x9]
; CHECK-NEXT:    ldp d1, d0, [sp, #8]
; CHECK-NEXT:    mov v1.d[1], v0.d[0]
; CHECK-NEXT:    ldr d0, [sp, #24]
; CHECK-NEXT:    str d0, [x8, #16]
; CHECK-NEXT:    str q1, [x8]
; CHECK-NEXT:    add sp, sp, #32
; CHECK-NEXT:    ret
  %retval = call <24 x i8> @llvm.vector.interleave3.v24i8(<8 x i8> %vec0, <8 x i8> %vec1, <8 x i8> %vec2)
  ret <24 x i8> %retval
}