| ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py |
| ; RUN: llc -mtriple=arm-eabi -mattr=+v8.6a,+neon -float-abi=hard < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16 |
| ; RUN: llc -mtriple=arm-eabi -mattr=+v8.6a,+neon,+bf16 -float-abi=hard < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16 |
| ; RUN: llc -mtriple=arm-eabi -mattr=+v8.6a,+neon,+fullfp16,+bf16 -float-abi=hard < %s | FileCheck %s --check-prefixes=CHECK,CHECK-FP16 |
| |
| %struct.float16x4x2_t = type { [2 x <4 x bfloat>] } |
| %struct.float16x8x2_t = type { [2 x <8 x bfloat>] } |
| |
| define dso_local <4 x bfloat> @test_vbsl_bf16(<4 x i16> %a, <4 x bfloat> %b, <4 x bfloat> %c) { |
| ; CHECK-LABEL: test_vbsl_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vbsl d0, d1, d2 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %0 = bitcast <4 x i16> %a to <8 x i8> |
| %1 = bitcast <4 x bfloat> %b to <8 x i8> |
| %2 = bitcast <4 x bfloat> %c to <8 x i8> |
| %vbsl_v.i = tail call <8 x i8> @llvm.arm.neon.vbsl.v8i8(<8 x i8> %0, <8 x i8> %1, <8 x i8> %2) |
| %3 = bitcast <8 x i8> %vbsl_v.i to <4 x bfloat> |
| ret <4 x bfloat> %3 |
| } |
| |
| define dso_local <8 x bfloat> @test_vbslq_bf16(<8 x i16> %a, <8 x bfloat> %b, <8 x bfloat> %c) { |
| ; CHECK-LABEL: test_vbslq_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vbsl q0, q1, q2 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %0 = bitcast <8 x i16> %a to <16 x i8> |
| %1 = bitcast <8 x bfloat> %b to <16 x i8> |
| %2 = bitcast <8 x bfloat> %c to <16 x i8> |
| %vbslq_v.i = tail call <16 x i8> @llvm.arm.neon.vbsl.v16i8(<16 x i8> %0, <16 x i8> %1, <16 x i8> %2) |
| %3 = bitcast <16 x i8> %vbslq_v.i to <8 x bfloat> |
| ret <8 x bfloat> %3 |
| } |
| |
| define dso_local %struct.float16x4x2_t @test_vzip_bf16(<4 x bfloat> %a, <4 x bfloat> %b) { |
| ; CHECK-LABEL: test_vzip_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vzip.16 d0, d1 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %vzip.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 4, i32 1, i32 5> |
| %vzip1.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 2, i32 6, i32 3, i32 7> |
| %.fca.0.0.insert = insertvalue %struct.float16x4x2_t undef, <4 x bfloat> %vzip.i, 0, 0 |
| %.fca.0.1.insert = insertvalue %struct.float16x4x2_t %.fca.0.0.insert, <4 x bfloat> %vzip1.i, 0, 1 |
| ret %struct.float16x4x2_t %.fca.0.1.insert |
| } |
| |
| define dso_local %struct.float16x8x2_t @test_vzipq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) { |
| ; CHECK-LABEL: test_vzipq_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vzip.16 q0, q1 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %vzip.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11> |
| %vzip1.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15> |
| %.fca.0.0.insert = insertvalue %struct.float16x8x2_t undef, <8 x bfloat> %vzip.i, 0, 0 |
| %.fca.0.1.insert = insertvalue %struct.float16x8x2_t %.fca.0.0.insert, <8 x bfloat> %vzip1.i, 0, 1 |
| ret %struct.float16x8x2_t %.fca.0.1.insert |
| } |
| |
| define dso_local %struct.float16x4x2_t @test_vuzp_bf16(<4 x bfloat> %a, <4 x bfloat> %b) { |
| ; CHECK-LABEL: test_vuzp_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vuzp.16 d0, d1 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %vuzp.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 6> |
| %vuzp1.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 1, i32 3, i32 5, i32 7> |
| %.fca.0.0.insert = insertvalue %struct.float16x4x2_t undef, <4 x bfloat> %vuzp.i, 0, 0 |
| %.fca.0.1.insert = insertvalue %struct.float16x4x2_t %.fca.0.0.insert, <4 x bfloat> %vuzp1.i, 0, 1 |
| ret %struct.float16x4x2_t %.fca.0.1.insert |
| } |
| |
| define dso_local %struct.float16x8x2_t @test_vuzpq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) { |
| ; CHECK-LABEL: test_vuzpq_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vuzp.16 q0, q1 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %vuzp.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14> |
| %vuzp1.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15> |
| %.fca.0.0.insert = insertvalue %struct.float16x8x2_t undef, <8 x bfloat> %vuzp.i, 0, 0 |
| %.fca.0.1.insert = insertvalue %struct.float16x8x2_t %.fca.0.0.insert, <8 x bfloat> %vuzp1.i, 0, 1 |
| ret %struct.float16x8x2_t %.fca.0.1.insert |
| } |
| |
| define dso_local %struct.float16x4x2_t @test_vtrn_bf16(<4 x bfloat> %a, <4 x bfloat> %b) { |
| ; CHECK-LABEL: test_vtrn_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vtrn.16 d0, d1 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %vtrn.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 4, i32 2, i32 6> |
| %vtrn1.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 1, i32 5, i32 3, i32 7> |
| %.fca.0.0.insert = insertvalue %struct.float16x4x2_t undef, <4 x bfloat> %vtrn.i, 0, 0 |
| %.fca.0.1.insert = insertvalue %struct.float16x4x2_t %.fca.0.0.insert, <4 x bfloat> %vtrn1.i, 0, 1 |
| ret %struct.float16x4x2_t %.fca.0.1.insert |
| } |
| |
| define dso_local %struct.float16x8x2_t @test_vtrnq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) { |
| ; CHECK-LABEL: test_vtrnq_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vtrn.16 q0, q1 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %vtrn.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14> |
| %vtrn1.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15> |
| %.fca.0.0.insert = insertvalue %struct.float16x8x2_t undef, <8 x bfloat> %vtrn.i, 0, 0 |
| %.fca.0.1.insert = insertvalue %struct.float16x8x2_t %.fca.0.0.insert, <8 x bfloat> %vtrn1.i, 0, 1 |
| ret %struct.float16x8x2_t %.fca.0.1.insert |
| } |
| |
| define dso_local <4 x bfloat> @test_vmov_n_bf16(float %a.coerce) { |
| ; CHECK-NOFP16-LABEL: test_vmov_n_bf16: |
| ; CHECK-NOFP16: @ %bb.0: @ %entry |
| ; CHECK-NOFP16-NEXT: .pad #4 |
| ; CHECK-NOFP16-NEXT: sub sp, sp, #4 |
| ; CHECK-NOFP16-NEXT: vmov r0, s0 |
| ; CHECK-NOFP16-NEXT: strh r0, [sp, #2] |
| ; CHECK-NOFP16-NEXT: add r0, sp, #2 |
| ; CHECK-NOFP16-NEXT: vld1.16 {d0[]}, [r0:16] |
| ; CHECK-NOFP16-NEXT: add sp, sp, #4 |
| ; CHECK-NOFP16-NEXT: bx lr |
| ; |
| ; CHECK-FP16-LABEL: test_vmov_n_bf16: |
| ; CHECK-FP16: @ %bb.0: @ %entry |
| ; CHECK-FP16-NEXT: @ kill: def $s0 killed $s0 def $d0 |
| ; CHECK-FP16-NEXT: vdup.16 d0, d0[0] |
| ; CHECK-FP16-NEXT: bx lr |
| entry: |
| %0 = bitcast float %a.coerce to i32 |
| %tmp.0.extract.trunc = trunc i32 %0 to i16 |
| %1 = bitcast i16 %tmp.0.extract.trunc to bfloat |
| %vecinit = insertelement <4 x bfloat> undef, bfloat %1, i32 0 |
| %vecinit4 = shufflevector <4 x bfloat> %vecinit, <4 x bfloat> undef, <4 x i32> zeroinitializer |
| ret <4 x bfloat> %vecinit4 |
| } |
| |
| define dso_local <8 x bfloat> @test_vmovq_n_bf16(float %a.coerce) { |
| ; CHECK-NOFP16-LABEL: test_vmovq_n_bf16: |
| ; CHECK-NOFP16: @ %bb.0: @ %entry |
| ; CHECK-NOFP16-NEXT: .pad #4 |
| ; CHECK-NOFP16-NEXT: sub sp, sp, #4 |
| ; CHECK-NOFP16-NEXT: vmov r0, s0 |
| ; CHECK-NOFP16-NEXT: strh r0, [sp, #2] |
| ; CHECK-NOFP16-NEXT: add r0, sp, #2 |
| ; CHECK-NOFP16-NEXT: vld1.16 {d0[], d1[]}, [r0:16] |
| ; CHECK-NOFP16-NEXT: add sp, sp, #4 |
| ; CHECK-NOFP16-NEXT: bx lr |
| ; |
| ; CHECK-FP16-LABEL: test_vmovq_n_bf16: |
| ; CHECK-FP16: @ %bb.0: @ %entry |
| ; CHECK-FP16-NEXT: @ kill: def $s0 killed $s0 def $d0 |
| ; CHECK-FP16-NEXT: vdup.16 q0, d0[0] |
| ; CHECK-FP16-NEXT: bx lr |
| entry: |
| %0 = bitcast float %a.coerce to i32 |
| %tmp.0.extract.trunc = trunc i32 %0 to i16 |
| %1 = bitcast i16 %tmp.0.extract.trunc to bfloat |
| %vecinit = insertelement <8 x bfloat> undef, bfloat %1, i32 0 |
| %vecinit8 = shufflevector <8 x bfloat> %vecinit, <8 x bfloat> undef, <8 x i32> zeroinitializer |
| ret <8 x bfloat> %vecinit8 |
| } |
| |
| define dso_local <4 x bfloat> @test_vdup_n_bf16(float %a.coerce) { |
| ; CHECK-NOFP16-LABEL: test_vdup_n_bf16: |
| ; CHECK-NOFP16: @ %bb.0: @ %entry |
| ; CHECK-NOFP16-NEXT: .pad #4 |
| ; CHECK-NOFP16-NEXT: sub sp, sp, #4 |
| ; CHECK-NOFP16-NEXT: vmov r0, s0 |
| ; CHECK-NOFP16-NEXT: strh r0, [sp, #2] |
| ; CHECK-NOFP16-NEXT: add r0, sp, #2 |
| ; CHECK-NOFP16-NEXT: vld1.16 {d0[]}, [r0:16] |
| ; CHECK-NOFP16-NEXT: add sp, sp, #4 |
| ; CHECK-NOFP16-NEXT: bx lr |
| ; |
| ; CHECK-FP16-LABEL: test_vdup_n_bf16: |
| ; CHECK-FP16: @ %bb.0: @ %entry |
| ; CHECK-FP16-NEXT: @ kill: def $s0 killed $s0 def $d0 |
| ; CHECK-FP16-NEXT: vdup.16 d0, d0[0] |
| ; CHECK-FP16-NEXT: bx lr |
| entry: |
| %0 = bitcast float %a.coerce to i32 |
| %tmp.0.extract.trunc = trunc i32 %0 to i16 |
| %1 = bitcast i16 %tmp.0.extract.trunc to bfloat |
| %vecinit = insertelement <4 x bfloat> undef, bfloat %1, i32 0 |
| %vecinit4 = shufflevector <4 x bfloat> %vecinit, <4 x bfloat> undef, <4 x i32> zeroinitializer |
| ret <4 x bfloat> %vecinit4 |
| } |
| |
| define dso_local <8 x bfloat> @test_vdupq_n_bf16(float %a.coerce) { |
| ; CHECK-NOFP16-LABEL: test_vdupq_n_bf16: |
| ; CHECK-NOFP16: @ %bb.0: @ %entry |
| ; CHECK-NOFP16-NEXT: .pad #4 |
| ; CHECK-NOFP16-NEXT: sub sp, sp, #4 |
| ; CHECK-NOFP16-NEXT: vmov r0, s0 |
| ; CHECK-NOFP16-NEXT: strh r0, [sp, #2] |
| ; CHECK-NOFP16-NEXT: add r0, sp, #2 |
| ; CHECK-NOFP16-NEXT: vld1.16 {d0[], d1[]}, [r0:16] |
| ; CHECK-NOFP16-NEXT: add sp, sp, #4 |
| ; CHECK-NOFP16-NEXT: bx lr |
| ; |
| ; CHECK-FP16-LABEL: test_vdupq_n_bf16: |
| ; CHECK-FP16: @ %bb.0: @ %entry |
| ; CHECK-FP16-NEXT: @ kill: def $s0 killed $s0 def $d0 |
| ; CHECK-FP16-NEXT: vdup.16 q0, d0[0] |
| ; CHECK-FP16-NEXT: bx lr |
| entry: |
| %0 = bitcast float %a.coerce to i32 |
| %tmp.0.extract.trunc = trunc i32 %0 to i16 |
| %1 = bitcast i16 %tmp.0.extract.trunc to bfloat |
| %vecinit = insertelement <8 x bfloat> undef, bfloat %1, i32 0 |
| %vecinit8 = shufflevector <8 x bfloat> %vecinit, <8 x bfloat> undef, <8 x i32> zeroinitializer |
| ret <8 x bfloat> %vecinit8 |
| } |
| |
| define dso_local <4 x bfloat> @test_vdup_lane_bf16(<4 x bfloat> %a) { |
| ; CHECK-LABEL: test_vdup_lane_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vdup.16 d0, d0[3] |
| ; CHECK-NEXT: bx lr |
| entry: |
| %shuffle = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3> |
| ret <4 x bfloat> %shuffle |
| } |
| |
| define dso_local <8 x bfloat> @test_vdupq_lane_bf16(<4 x bfloat> %a) { |
| ; CHECK-LABEL: test_vdupq_lane_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: @ kill: def $d0 killed $d0 def $q0 |
| ; CHECK-NEXT: vdup.16 q0, d0[3] |
| ; CHECK-NEXT: bx lr |
| entry: |
| %shuffle = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3> |
| ret <8 x bfloat> %shuffle |
| } |
| |
| define dso_local <4 x bfloat> @test_vext_bf16(<4 x bfloat> %a, <4 x bfloat> %b) { |
| ; CHECK-LABEL: test_vext_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vext.16 d0, d0, d1, #2 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %vext = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 2, i32 3, i32 4, i32 5> |
| ret <4 x bfloat> %vext |
| } |
| |
| define dso_local <8 x bfloat> @test_vextq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) { |
| ; CHECK-LABEL: test_vextq_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vext.16 q0, q0, q1, #5 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %vext = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12> |
| ret <8 x bfloat> %vext |
| } |
| |
| define dso_local <4 x bfloat> @test_vext_aligned_bf16(<8 x bfloat> %a) { |
| ; CHECK-LABEL: test_vext_aligned_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vmov.f64 d0, d1 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %vext = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7> |
| ret <4 x bfloat> %vext |
| } |
| |
| define dso_local <4 x bfloat> @test_vext_unaligned_bf16(<8 x bfloat> %a) { |
| ; CHECK-LABEL: test_vext_unaligned_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vext.16 d0, d0, d1, #3 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %vext = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <4 x i32> <i32 3, i32 4, i32 5, i32 6> |
| ret <4 x bfloat> %vext |
| } |
| |
| define arm_aapcs_vfpcc <8 x bfloat> @shuffle3step0_bf16(<32 x bfloat> %src) { |
| ; CHECK-NOFP16-LABEL: shuffle3step0_bf16: |
| ; CHECK-NOFP16: @ %bb.0: @ %entry |
| ; CHECK-NOFP16-NEXT: vmov r1, s0 |
| ; CHECK-NOFP16-NEXT: vmov.u16 r0, d0[3] |
| ; CHECK-NOFP16-NEXT: vrev32.16 d16, d3 |
| ; CHECK-NOFP16-NEXT: vext.16 d17, d4, d5, #2 |
| ; CHECK-NOFP16-NEXT: vext.16 d16, d16, d3, #1 |
| ; CHECK-NOFP16-NEXT: vext.16 d16, d17, d16, #2 |
| ; CHECK-NOFP16-NEXT: vext.16 d16, d16, d17, #1 |
| ; CHECK-NOFP16-NEXT: vext.16 d17, d16, d16, #1 |
| ; CHECK-NOFP16-NEXT: vmov.16 d16[0], r1 |
| ; CHECK-NOFP16-NEXT: vmov.16 d16[1], r0 |
| ; CHECK-NOFP16-NEXT: vmov r0, s3 |
| ; CHECK-NOFP16-NEXT: vmov.16 d16[2], r0 |
| ; CHECK-NOFP16-NEXT: vmov.u16 r0, d2[1] |
| ; CHECK-NOFP16-NEXT: vmov.16 d16[3], r0 |
| ; CHECK-NOFP16-NEXT: vorr q0, q8, q8 |
| ; CHECK-NOFP16-NEXT: bx lr |
| ; |
| ; CHECK-FP16-LABEL: shuffle3step0_bf16: |
| ; CHECK-FP16: @ %bb.0: @ %entry |
| ; CHECK-FP16-NEXT: vmov r1, s0 |
| ; CHECK-FP16-NEXT: vext.16 d17, d4, d5, #2 |
| ; CHECK-FP16-NEXT: vmovx.f16 s8, s1 |
| ; CHECK-FP16-NEXT: vrev32.16 d16, d3 |
| ; CHECK-FP16-NEXT: vmov r0, s8 |
| ; CHECK-FP16-NEXT: vext.16 d16, d16, d3, #1 |
| ; CHECK-FP16-NEXT: vext.16 d16, d17, d16, #2 |
| ; CHECK-FP16-NEXT: vext.16 d16, d16, d17, #1 |
| ; CHECK-FP16-NEXT: vext.16 d17, d16, d16, #1 |
| ; CHECK-FP16-NEXT: vmov.16 d16[0], r1 |
| ; CHECK-FP16-NEXT: vmov.16 d16[1], r0 |
| ; CHECK-FP16-NEXT: vmov r0, s3 |
| ; CHECK-FP16-NEXT: vmovx.f16 s0, s4 |
| ; CHECK-FP16-NEXT: vmov.16 d16[2], r0 |
| ; CHECK-FP16-NEXT: vmov r0, s0 |
| ; CHECK-FP16-NEXT: vmov.16 d16[3], r0 |
| ; CHECK-FP16-NEXT: vorr q0, q8, q8 |
| ; CHECK-FP16-NEXT: bx lr |
| entry: |
| %s1 = shufflevector <32 x bfloat> %src, <32 x bfloat> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21> |
| ret <8 x bfloat> %s1 |
| } |
| |
| define arm_aapcs_vfpcc <8 x bfloat> @shuffle3step1_bf16(<32 x bfloat> %src) { |
| ; CHECK-NOFP16-LABEL: shuffle3step1_bf16: |
| ; CHECK-NOFP16: @ %bb.0: @ %entry |
| ; CHECK-NOFP16-NEXT: vorr q3, q0, q0 |
| ; CHECK-NOFP16-NEXT: vmov.u16 r1, d6[1] |
| ; CHECK-NOFP16-NEXT: vmov r0, s14 |
| ; CHECK-NOFP16-NEXT: vmov.16 d0[0], r1 |
| ; CHECK-NOFP16-NEXT: vmov.16 d0[1], r0 |
| ; CHECK-NOFP16-NEXT: vmov.u16 r0, d7[3] |
| ; CHECK-NOFP16-NEXT: vmov.16 d0[2], r0 |
| ; CHECK-NOFP16-NEXT: vmov r0, s5 |
| ; CHECK-NOFP16-NEXT: vdup.16 q1, d3[1] |
| ; CHECK-NOFP16-NEXT: vmov r1, s4 |
| ; CHECK-NOFP16-NEXT: vmov.16 d0[3], r0 |
| ; CHECK-NOFP16-NEXT: vmov r0, s8 |
| ; CHECK-NOFP16-NEXT: vmov.16 d1[0], r1 |
| ; CHECK-NOFP16-NEXT: vmov.16 d1[1], r0 |
| ; CHECK-NOFP16-NEXT: vmov.u16 r0, d4[3] |
| ; CHECK-NOFP16-NEXT: vmov.16 d1[2], r0 |
| ; CHECK-NOFP16-NEXT: vmov r0, s11 |
| ; CHECK-NOFP16-NEXT: vmov.16 d1[3], r0 |
| ; CHECK-NOFP16-NEXT: bx lr |
| ; |
| ; CHECK-FP16-LABEL: shuffle3step1_bf16: |
| ; CHECK-FP16: @ %bb.0: @ %entry |
| ; CHECK-FP16-NEXT: vorr q3, q0, q0 |
| ; CHECK-FP16-NEXT: vmovx.f16 s0, s12 |
| ; CHECK-FP16-NEXT: vmovx.f16 s12, s15 |
| ; CHECK-FP16-NEXT: vmov r1, s0 |
| ; CHECK-FP16-NEXT: vmov r0, s14 |
| ; CHECK-FP16-NEXT: vmov.16 d0[0], r1 |
| ; CHECK-FP16-NEXT: vmov.16 d0[1], r0 |
| ; CHECK-FP16-NEXT: vmov r0, s12 |
| ; CHECK-FP16-NEXT: vmov.16 d0[2], r0 |
| ; CHECK-FP16-NEXT: vmov r0, s5 |
| ; CHECK-FP16-NEXT: vdup.16 q1, d3[1] |
| ; CHECK-FP16-NEXT: vmov r1, s4 |
| ; CHECK-FP16-NEXT: vmovx.f16 s4, s9 |
| ; CHECK-FP16-NEXT: vmov.16 d0[3], r0 |
| ; CHECK-FP16-NEXT: vmov r0, s8 |
| ; CHECK-FP16-NEXT: vmov.16 d1[0], r1 |
| ; CHECK-FP16-NEXT: vmov.16 d1[1], r0 |
| ; CHECK-FP16-NEXT: vmov r0, s4 |
| ; CHECK-FP16-NEXT: vmov.16 d1[2], r0 |
| ; CHECK-FP16-NEXT: vmov r0, s11 |
| ; CHECK-FP16-NEXT: vmov.16 d1[3], r0 |
| ; CHECK-FP16-NEXT: bx lr |
| entry: |
| %s1 = shufflevector <32 x bfloat> %src, <32 x bfloat> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22> |
| ret <8 x bfloat> %s1 |
| } |
| |
| define arm_aapcs_vfpcc <8 x bfloat> @shuffle3step2_bf16(<32 x bfloat> %src) { |
| ; CHECK-NOFP16-LABEL: shuffle3step2_bf16: |
| ; CHECK-NOFP16: @ %bb.0: @ %entry |
| ; CHECK-NOFP16-NEXT: vext.16 d16, d0, d1, #2 |
| ; CHECK-NOFP16-NEXT: vmov.u16 r0, d4[1] |
| ; CHECK-NOFP16-NEXT: vext.16 d17, d16, d2, #3 |
| ; CHECK-NOFP16-NEXT: vext.16 d16, d2, d16, #1 |
| ; CHECK-NOFP16-NEXT: vdup.16 q1, d3[2] |
| ; CHECK-NOFP16-NEXT: vext.16 d16, d16, d17, #2 |
| ; CHECK-NOFP16-NEXT: vmov r1, s4 |
| ; CHECK-NOFP16-NEXT: vext.16 d0, d16, d16, #1 |
| ; CHECK-NOFP16-NEXT: vmov.16 d1[0], r1 |
| ; CHECK-NOFP16-NEXT: vmov.16 d1[1], r0 |
| ; CHECK-NOFP16-NEXT: vmov r0, s10 |
| ; CHECK-NOFP16-NEXT: vmov.16 d1[2], r0 |
| ; CHECK-NOFP16-NEXT: vmov.u16 r0, d5[3] |
| ; CHECK-NOFP16-NEXT: vmov.16 d1[3], r0 |
| ; CHECK-NOFP16-NEXT: bx lr |
| ; |
| ; CHECK-FP16-LABEL: shuffle3step2_bf16: |
| ; CHECK-FP16: @ %bb.0: @ %entry |
| ; CHECK-FP16-NEXT: vext.16 d16, d0, d1, #2 |
| ; CHECK-FP16-NEXT: vmovx.f16 s12, s8 |
| ; CHECK-FP16-NEXT: vmov r0, s12 |
| ; CHECK-FP16-NEXT: vext.16 d17, d16, d2, #3 |
| ; CHECK-FP16-NEXT: vext.16 d16, d2, d16, #1 |
| ; CHECK-FP16-NEXT: vdup.16 q1, d3[2] |
| ; CHECK-FP16-NEXT: vext.16 d16, d16, d17, #2 |
| ; CHECK-FP16-NEXT: vmov r1, s4 |
| ; CHECK-FP16-NEXT: vmovx.f16 s4, s11 |
| ; CHECK-FP16-NEXT: vext.16 d0, d16, d16, #1 |
| ; CHECK-FP16-NEXT: vmov.16 d1[0], r1 |
| ; CHECK-FP16-NEXT: vmov.16 d1[1], r0 |
| ; CHECK-FP16-NEXT: vmov r0, s10 |
| ; CHECK-FP16-NEXT: vmov.16 d1[2], r0 |
| ; CHECK-FP16-NEXT: vmov r0, s4 |
| ; CHECK-FP16-NEXT: vmov.16 d1[3], r0 |
| ; CHECK-FP16-NEXT: bx lr |
| entry: |
| %s1 = shufflevector <32 x bfloat> %src, <32 x bfloat> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23> |
| ret <8 x bfloat> %s1 |
| } |
| |
| |
| define dso_local <4 x bfloat> @test_vrev64_bf16(<4 x bfloat> %a) { |
| ; CHECK-LABEL: test_vrev64_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vrev64.16 d0, d0 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %shuffle.i = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0> |
| ret <4 x bfloat> %shuffle.i |
| } |
| |
| define dso_local <8 x bfloat> @test_vrev64q_bf16(<8 x bfloat> %a) { |
| ; CHECK-LABEL: test_vrev64q_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vrev64.16 q0, q0 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %shuffle.i = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4> |
| ret <8 x bfloat> %shuffle.i |
| } |
| |
| define dso_local <4 x bfloat> @test_vrev32_bf16(<4 x bfloat> %a) { |
| ; CHECK-LABEL: test_vrev32_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vrev32.16 d0, d0 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %shuffle.i = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2> |
| ret <4 x bfloat> %shuffle.i |
| } |
| |
| define dso_local <8 x bfloat> @test_vrev32q_bf16(<8 x bfloat> %a) { |
| ; CHECK-LABEL: test_vrev32q_bf16: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vrev32.16 q0, q0 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %shuffle.i = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6> |
| ret <8 x bfloat> %shuffle.i |
| } |
| |
| define <4 x bfloat> @test_vld_dup1_4xbfloat(ptr %b) { |
| ; CHECK-LABEL: test_vld_dup1_4xbfloat: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vld1.16 {d0[]}, [r0:16] |
| ; CHECK-NEXT: bx lr |
| entry: |
| %b1 = load bfloat, ptr %b, align 2 |
| %vecinit = insertelement <4 x bfloat> undef, bfloat %b1, i32 0 |
| %vecinit2 = insertelement <4 x bfloat> %vecinit, bfloat %b1, i32 1 |
| %vecinit3 = insertelement <4 x bfloat> %vecinit2, bfloat %b1, i32 2 |
| %vecinit4 = insertelement <4 x bfloat> %vecinit3, bfloat %b1, i32 3 |
| ret <4 x bfloat> %vecinit4 |
| } |
| |
| define <8 x bfloat> @test_vld_dup1_8xbfloat(ptr %b) local_unnamed_addr { |
| ; CHECK-LABEL: test_vld_dup1_8xbfloat: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: vld1.16 {d0[], d1[]}, [r0:16] |
| ; CHECK-NEXT: bx lr |
| entry: |
| %b1 = load bfloat, ptr %b, align 2 |
| %vecinit = insertelement <8 x bfloat> undef, bfloat %b1, i32 0 |
| %vecinit8 = shufflevector <8 x bfloat> %vecinit, <8 x bfloat> undef, <8 x i32> zeroinitializer |
| ret <8 x bfloat> %vecinit8 |
| } |
| |
| define <8 x bfloat> @test_shufflevector8xbfloat(<4 x bfloat> %a) { |
| ; CHECK-LABEL: test_shufflevector8xbfloat: |
| ; CHECK: @ %bb.0: @ %entry |
| ; CHECK-NEXT: @ kill: def $d0 killed $d0 def $q0 |
| ; CHECK-NEXT: vmov.f64 d1, d0 |
| ; CHECK-NEXT: bx lr |
| entry: |
| %r = shufflevector <4 x bfloat> %a, <4 x bfloat> %a, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> |
| ret <8 x bfloat> %r |
| } |
| |
| declare <8 x i8> @llvm.arm.neon.vbsl.v8i8(<8 x i8>, <8 x i8>, <8 x i8>) |
| declare <16 x i8> @llvm.arm.neon.vbsl.v16i8(<16 x i8>, <16 x i8>, <16 x i8>) |