Google Git
Sign in
llvm/llvm-project/f9d3278901cfef8e6a998eb9ebd17d97fc31291e/./llvm/test/CodeGen/AArch64/aarch64-reassociate-accumulators-sve.ll
blob: 5547567092903ef39cfa0f3bb4e61221bb9faed6 [file]
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "aba" --filter "abd" --filter "add" --version 5
; RUN: opt -passes=loop-unroll %s -o - | llc -O3 - -mtriple=aarch64-unknown-unknown -mcpu=neoverse-v2 -o - | FileCheck %s
define i64 @sabalb_i32_to_i64_accumulation(ptr %ptr1, ptr %ptr2) {
; CHECK-LABEL: sabalb_i32_to_i64_accumulation:
; CHECK: sabdlb z1.d, z1.s, z2.s
; CHECK: sabalb z0.d, z3.s, z4.s
; CHECK: sabalb z1.d, z4.s, z2.s
; CHECK: sabdlb z2.d, z3.s, z2.s
; CHECK: sabalb z2.d, z4.s, z3.s
; CHECK: sabalb z0.d, z3.s, z4.s
; CHECK: sabalb z1.d, z3.s, z4.s
; CHECK: sabalb z2.d, z3.s, z4.s
; CHECK: sabalb z0.d, z3.s, z4.s
; CHECK: sabalb z1.d, z3.s, z4.s
; CHECK: sabalb z2.d, z3.s, z4.s
; CHECK: sabalb z0.d, z3.s, z4.s
; CHECK: sabalb z1.d, z3.s, z4.s
; CHECK: sabalb z2.d, z3.s, z4.s
; CHECK: sabalb z0.d, z3.s, z4.s
; CHECK: add z0.d, z2.d, z0.d
; CHECK: sabalb z1.d, z3.s, z4.s
; CHECK: add z0.d, z0.d, z1.d
; CHECK: uaddv d0, p0, z0.d
entry:
br label %loop
loop:
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
%acc_phi = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
%ptr1_i = getelementptr <vscale x 4 x i32>, ptr %ptr1, i32 %i
%ptr2_i = getelementptr <vscale x 4 x i32>, ptr %ptr2, i32 %i
%a = load <vscale x 4 x i32>, ptr %ptr1_i, align 1
%b = load <vscale x 4 x i32>, ptr %ptr2_i, align 1
%acc_next = call <vscale x 2 x i64> @llvm.aarch64.sve.sabalb.nxv2i64(<vscale x 2 x i64> %acc_phi,
<vscale x 4 x i32> %a,
<vscale x 4 x i32> %b)
%next_i = add i32 %i, 1
%cmp = icmp slt i32 %next_i, 16
br i1 %cmp, label %loop, label %exit
exit:
%reduce = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %acc_next)
ret i64 %reduce
}
declare <vscale x 2 x i64> @llvm.aarch64.sve.sabalb.nxv2i64(<vscale x 2 x i64>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64>)
define i32 @sabalb_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {
; CHECK-LABEL: sabalb_i16_to_i32_accumulation:
; CHECK: sabdlb z1.s, z1.h, z2.h
; CHECK: sabalb z0.s, z3.h, z4.h
; CHECK: sabalb z1.s, z4.h, z2.h
; CHECK: sabdlb z2.s, z3.h, z2.h
; CHECK: sabalb z2.s, z4.h, z3.h
; CHECK: sabalb z0.s, z3.h, z4.h
; CHECK: sabalb z1.s, z3.h, z4.h
; CHECK: sabalb z2.s, z3.h, z4.h
; CHECK: sabalb z0.s, z3.h, z4.h
; CHECK: sabalb z1.s, z3.h, z4.h
; CHECK: sabalb z2.s, z3.h, z4.h
; CHECK: sabalb z0.s, z3.h, z4.h
; CHECK: sabalb z1.s, z3.h, z4.h
; CHECK: sabalb z2.s, z3.h, z4.h
; CHECK: sabalb z0.s, z3.h, z4.h
; CHECK: add z0.s, z2.s, z0.s
; CHECK: sabalb z1.s, z3.h, z4.h
; CHECK: add z0.s, z0.s, z1.s
; CHECK: uaddv d0, p0, z0.s
entry:
br label %loop
loop:
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
%acc_phi = phi <vscale x 4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
%ptr1_i = getelementptr <vscale x 8 x i16>, ptr %ptr1, i32 %i
%ptr2_i = getelementptr <vscale x 8 x i16>, ptr %ptr2, i32 %i
%a = load <vscale x 8 x i16>, ptr %ptr1_i, align 1
%b = load <vscale x 8 x i16>, ptr %ptr2_i, align 1
%acc_next = call <vscale x 4 x i32> @llvm.aarch64.sve.sabalb.nxv4i32(<vscale x 4 x i32> %acc_phi,
<vscale x 8 x i16> %a,
<vscale x 8 x i16> %b)
%next_i = add i32 %i, 1
%cmp = icmp slt i32 %next_i, 16
br i1 %cmp, label %loop, label %exit
exit:
%reduce = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %acc_next)
ret i32 %reduce
}
declare <vscale x 4 x i32> @llvm.aarch64.sve.sabalb.nxv4i32(<vscale x 4 x i32>, <vscale x 8 x i16>, <vscale x 8 x i16>)
declare i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32>)
define i16 @sabalb_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {
; CHECK-LABEL: sabalb_i8_to_i16_accumulation:
; CHECK: sabdlb z1.h, z1.b, z2.b
; CHECK: sabalb z0.h, z3.b, z4.b
; CHECK: sabalb z1.h, z4.b, z2.b
; CHECK: sabdlb z2.h, z3.b, z2.b
; CHECK: sabalb z2.h, z4.b, z3.b
; CHECK: sabalb z0.h, z3.b, z4.b
; CHECK: sabalb z1.h, z3.b, z4.b
; CHECK: sabalb z2.h, z3.b, z4.b
; CHECK: sabalb z0.h, z3.b, z4.b
; CHECK: sabalb z1.h, z3.b, z4.b
; CHECK: sabalb z2.h, z3.b, z4.b
; CHECK: sabalb z0.h, z3.b, z4.b
; CHECK: sabalb z1.h, z3.b, z4.b
; CHECK: sabalb z2.h, z3.b, z4.b
; CHECK: sabalb z0.h, z3.b, z4.b
; CHECK: add z0.h, z2.h, z0.h
; CHECK: sabalb z1.h, z3.b, z4.b
; CHECK: add z0.h, z0.h, z1.h
; CHECK: uaddv d0, p0, z0.h
entry:
br label %loop
loop:
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
%acc_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
%ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i
%ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i
%a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1
%b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1
%acc_next = call <vscale x 8 x i16> @llvm.aarch64.sve.sabalb.nxv8i16(<vscale x 8 x i16> %acc_phi,
<vscale x 16 x i8> %a,
<vscale x 16 x i8> %b)
%next_i = add i32 %i, 1
%cmp = icmp slt i32 %next_i, 16
br i1 %cmp, label %loop, label %exit
exit:
%reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)
ret i16 %reduce
}
declare <vscale x 8 x i16> @llvm.aarch64.sve.sabalb.nxv8i16(<vscale x 8 x i16>, <vscale x 16 x i8>, <vscale x 16 x i8>)
declare i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16>)
define i64 @sabalt_i32_to_i64_accumulation(ptr %ptr1, ptr %ptr2) {
; CHECK-LABEL: sabalt_i32_to_i64_accumulation:
; CHECK: sabdlt z1.d, z1.s, z2.s
; CHECK: sabalt z0.d, z3.s, z4.s
; CHECK: sabalt z1.d, z4.s, z2.s
; CHECK: sabdlt z2.d, z3.s, z2.s
; CHECK: sabalt z2.d, z4.s, z3.s
; CHECK: sabalt z0.d, z3.s, z4.s
; CHECK: sabalt z1.d, z3.s, z4.s
; CHECK: sabalt z2.d, z3.s, z4.s
; CHECK: sabalt z0.d, z3.s, z4.s
; CHECK: sabalt z1.d, z3.s, z4.s
; CHECK: sabalt z2.d, z3.s, z4.s
; CHECK: sabalt z0.d, z3.s, z4.s
; CHECK: sabalt z1.d, z3.s, z4.s
; CHECK: sabalt z2.d, z3.s, z4.s
; CHECK: sabalt z0.d, z3.s, z4.s
; CHECK: add z0.d, z2.d, z0.d
; CHECK: sabalt z1.d, z3.s, z4.s
; CHECK: add z0.d, z0.d, z1.d
; CHECK: uaddv d0, p0, z0.d
entry:
br label %loop
loop:
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
%acc_phi = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
%ptr1_i = getelementptr <vscale x 4 x i32>, ptr %ptr1, i32 %i
%ptr2_i = getelementptr <vscale x 4 x i32>, ptr %ptr2, i32 %i
%a = load <vscale x 4 x i32>, ptr %ptr1_i, align 1
%b = load <vscale x 4 x i32>, ptr %ptr2_i, align 1
%acc_next = call <vscale x 2 x i64> @llvm.aarch64.sve.sabalt.nxv2i64(<vscale x 2 x i64> %acc_phi,
<vscale x 4 x i32> %a,
<vscale x 4 x i32> %b)
%next_i = add i32 %i, 1
%cmp = icmp slt i32 %next_i, 16
br i1 %cmp, label %loop, label %exit
exit:
%reduce = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %acc_next)
ret i64 %reduce
}
declare <vscale x 2 x i64> @llvm.aarch64.sve.sabalt.nxv2i64(<vscale x 2 x i64>, <vscale x 4 x i32>, <vscale x 4 x i32>)
define i32 @sabalt_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {
; CHECK-LABEL: sabalt_i16_to_i32_accumulation:
; CHECK: sabdlt z1.s, z1.h, z2.h
; CHECK: sabalt z0.s, z3.h, z4.h
; CHECK: sabalt z1.s, z4.h, z2.h
; CHECK: sabdlt z2.s, z3.h, z2.h
; CHECK: sabalt z2.s, z4.h, z3.h
; CHECK: sabalt z0.s, z3.h, z4.h
; CHECK: sabalt z1.s, z3.h, z4.h
; CHECK: sabalt z2.s, z3.h, z4.h
; CHECK: sabalt z0.s, z3.h, z4.h
; CHECK: sabalt z1.s, z3.h, z4.h
; CHECK: sabalt z2.s, z3.h, z4.h
; CHECK: sabalt z0.s, z3.h, z4.h
; CHECK: sabalt z1.s, z3.h, z4.h
; CHECK: sabalt z2.s, z3.h, z4.h
; CHECK: sabalt z0.s, z3.h, z4.h
; CHECK: add z0.s, z2.s, z0.s
; CHECK: sabalt z1.s, z3.h, z4.h
; CHECK: add z0.s, z0.s, z1.s
; CHECK: uaddv d0, p0, z0.s
entry:
br label %loop
loop:
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
%acc_phi = phi <vscale x 4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
%ptr1_i = getelementptr <vscale x 8 x i16>, ptr %ptr1, i32 %i
%ptr2_i = getelementptr <vscale x 8 x i16>, ptr %ptr2, i32 %i
%a = load <vscale x 8 x i16>, ptr %ptr1_i, align 1
%b = load <vscale x 8 x i16>, ptr %ptr2_i, align 1
%acc_next = call <vscale x 4 x i32> @llvm.aarch64.sve.sabalt.nxv4i32(<vscale x 4 x i32> %acc_phi,
<vscale x 8 x i16> %a,
<vscale x 8 x i16> %b)
%next_i = add i32 %i, 1
%cmp = icmp slt i32 %next_i, 16
br i1 %cmp, label %loop, label %exit
exit:
%reduce = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %acc_next)
ret i32 %reduce
}
declare <vscale x 4 x i32> @llvm.aarch64.sve.sabalt.nxv4i32(<vscale x 4 x i32>, <vscale x 8 x i16>, <vscale x 8 x i16>)
define i16 @sabalt_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {
; CHECK-LABEL: sabalt_i8_to_i16_accumulation:
; CHECK: sabdlt z1.h, z1.b, z2.b
; CHECK: sabalt z0.h, z3.b, z4.b
; CHECK: sabalt z1.h, z4.b, z2.b
; CHECK: sabdlt z2.h, z3.b, z2.b
; CHECK: sabalt z2.h, z4.b, z3.b
; CHECK: sabalt z0.h, z3.b, z4.b
; CHECK: sabalt z1.h, z3.b, z4.b
; CHECK: sabalt z2.h, z3.b, z4.b
; CHECK: sabalt z0.h, z3.b, z4.b
; CHECK: sabalt z1.h, z3.b, z4.b
; CHECK: sabalt z2.h, z3.b, z4.b
; CHECK: sabalt z0.h, z3.b, z4.b
; CHECK: sabalt z1.h, z3.b, z4.b
; CHECK: sabalt z2.h, z3.b, z4.b
; CHECK: sabalt z0.h, z3.b, z4.b
; CHECK: add z0.h, z2.h, z0.h
; CHECK: sabalt z1.h, z3.b, z4.b
; CHECK: add z0.h, z0.h, z1.h
; CHECK: uaddv d0, p0, z0.h
entry:
br label %loop
loop:
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
%acc_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
%ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i
%ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i
%a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1
%b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1
%acc_next = call <vscale x 8 x i16> @llvm.aarch64.sve.sabalt.nxv8i16(<vscale x 8 x i16> %acc_phi,
<vscale x 16 x i8> %a,
<vscale x 16 x i8> %b)
%next_i = add i32 %i, 1
%cmp = icmp slt i32 %next_i, 16
br i1 %cmp, label %loop, label %exit
exit:
%reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)
ret i16 %reduce
}
declare <vscale x 8 x i16> @llvm.aarch64.sve.sabalt.nxv8i16(<vscale x 8 x i16>, <vscale x 16 x i8>, <vscale x 16 x i8>)
define i64 @uabalb_i32_to_i64_accumulation(ptr %ptr1, ptr %ptr2) {
; CHECK-LABEL: uabalb_i32_to_i64_accumulation:
; CHECK: uabdlb z1.d, z1.s, z2.s
; CHECK: uabalb z0.d, z3.s, z4.s
; CHECK: uabalb z1.d, z4.s, z2.s
; CHECK: uabdlb z2.d, z3.s, z2.s
; CHECK: uabalb z2.d, z4.s, z3.s
; CHECK: uabalb z0.d, z3.s, z4.s
; CHECK: uabalb z1.d, z3.s, z4.s
; CHECK: uabalb z2.d, z3.s, z4.s
; CHECK: uabalb z0.d, z3.s, z4.s
; CHECK: uabalb z1.d, z3.s, z4.s
; CHECK: uabalb z2.d, z3.s, z4.s
; CHECK: uabalb z0.d, z3.s, z4.s
; CHECK: uabalb z1.d, z3.s, z4.s
; CHECK: uabalb z2.d, z3.s, z4.s
; CHECK: uabalb z0.d, z3.s, z4.s
; CHECK: add z0.d, z2.d, z0.d
; CHECK: uabalb z1.d, z3.s, z4.s
; CHECK: add z0.d, z0.d, z1.d
; CHECK: uaddv d0, p0, z0.d
entry:
br label %loop
loop:
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
%acc_phi = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
%ptr1_i = getelementptr <vscale x 4 x i32>, ptr %ptr1, i32 %i
%ptr2_i = getelementptr <vscale x 4 x i32>, ptr %ptr2, i32 %i
%a = load <vscale x 4 x i32>, ptr %ptr1_i, align 1
%b = load <vscale x 4 x i32>, ptr %ptr2_i, align 1
%acc_next = call <vscale x 2 x i64> @llvm.aarch64.sve.uabalb.nxv2i64(<vscale x 2 x i64> %acc_phi,
<vscale x 4 x i32> %a,
<vscale x 4 x i32> %b)
%next_i = add i32 %i, 1
%cmp = icmp slt i32 %next_i, 16
br i1 %cmp, label %loop, label %exit
exit:
%reduce = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %acc_next)
ret i64 %reduce
}
declare <vscale x 2 x i64> @llvm.aarch64.sve.uabalb.nxv2i64(<vscale x 2 x i64>, <vscale x 4 x i32>, <vscale x 4 x i32>)
define i32 @uabalb_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {
; CHECK-LABEL: uabalb_i16_to_i32_accumulation:
; CHECK: uabdlb z1.s, z1.h, z2.h
; CHECK: uabalb z0.s, z3.h, z4.h
; CHECK: uabalb z1.s, z4.h, z2.h
; CHECK: uabdlb z2.s, z3.h, z2.h
; CHECK: uabalb z2.s, z4.h, z3.h
; CHECK: uabalb z0.s, z3.h, z4.h
; CHECK: uabalb z1.s, z3.h, z4.h
; CHECK: uabalb z2.s, z3.h, z4.h
; CHECK: uabalb z0.s, z3.h, z4.h
; CHECK: uabalb z1.s, z3.h, z4.h
; CHECK: uabalb z2.s, z3.h, z4.h
; CHECK: uabalb z0.s, z3.h, z4.h
; CHECK: uabalb z1.s, z3.h, z4.h
; CHECK: uabalb z2.s, z3.h, z4.h
; CHECK: uabalb z0.s, z3.h, z4.h
; CHECK: add z0.s, z2.s, z0.s
; CHECK: uabalb z1.s, z3.h, z4.h
; CHECK: add z0.s, z0.s, z1.s
; CHECK: uaddv d0, p0, z0.s
entry:
br label %loop
loop:
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
%acc_phi = phi <vscale x 4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
%ptr1_i = getelementptr <vscale x 8 x i16>, ptr %ptr1, i32 %i
%ptr2_i = getelementptr <vscale x 8 x i16>, ptr %ptr2, i32 %i
%a = load <vscale x 8 x i16>, ptr %ptr1_i, align 1
%b = load <vscale x 8 x i16>, ptr %ptr2_i, align 1
%acc_next = call <vscale x 4 x i32> @llvm.aarch64.sve.uabalb.nxv4i32(<vscale x 4 x i32> %acc_phi,
<vscale x 8 x i16> %a,
<vscale x 8 x i16> %b)
%next_i = add i32 %i, 1
%cmp = icmp slt i32 %next_i, 16
br i1 %cmp, label %loop, label %exit
exit:
%reduce = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %acc_next)
ret i32 %reduce
}
declare <vscale x 4 x i32> @llvm.aarch64.sve.uabalb.nxv4i32(<vscale x 4 x i32>, <vscale x 8 x i16>, <vscale x 8 x i16>)
define i16 @uabalb_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {
; CHECK-LABEL: uabalb_i8_to_i16_accumulation:
; CHECK: uabdlb z1.h, z1.b, z2.b
; CHECK: uabalb z0.h, z3.b, z4.b
; CHECK: uabalb z1.h, z4.b, z2.b
; CHECK: uabdlb z2.h, z3.b, z2.b
; CHECK: uabalb z2.h, z4.b, z3.b
; CHECK: uabalb z0.h, z3.b, z4.b
; CHECK: uabalb z1.h, z3.b, z4.b
; CHECK: uabalb z2.h, z3.b, z4.b
; CHECK: uabalb z0.h, z3.b, z4.b
; CHECK: uabalb z1.h, z3.b, z4.b
; CHECK: uabalb z2.h, z3.b, z4.b
; CHECK: uabalb z0.h, z3.b, z4.b
; CHECK: uabalb z1.h, z3.b, z4.b
; CHECK: uabalb z2.h, z3.b, z4.b
; CHECK: uabalb z0.h, z3.b, z4.b
; CHECK: add z0.h, z2.h, z0.h
; CHECK: uabalb z1.h, z3.b, z4.b
; CHECK: add z0.h, z0.h, z1.h
; CHECK: uaddv d0, p0, z0.h
entry:
br label %loop
loop:
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
%acc_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
%ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i
%ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i
%a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1
%b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1
%acc_next = call <vscale x 8 x i16> @llvm.aarch64.sve.uabalb.nxv8i16(<vscale x 8 x i16> %acc_phi,
<vscale x 16 x i8> %a,
<vscale x 16 x i8> %b)
%next_i = add i32 %i, 1
%cmp = icmp slt i32 %next_i, 16
br i1 %cmp, label %loop, label %exit
exit:
%reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)
ret i16 %reduce
}
declare <vscale x 8 x i16> @llvm.aarch64.sve.uabalb.nxv8i16(<vscale x 8 x i16>, <vscale x 16 x i8>, <vscale x 16 x i8>)
define i64 @uabalt_i32_to_i64_accumulation(ptr %ptr1, ptr %ptr2) {
; CHECK-LABEL: uabalt_i32_to_i64_accumulation:
; CHECK: uabdlt z1.d, z1.s, z2.s
; CHECK: uabalt z0.d, z3.s, z4.s
; CHECK: uabalt z1.d, z4.s, z2.s
; CHECK: uabdlt z2.d, z3.s, z2.s
; CHECK: uabalt z2.d, z4.s, z3.s
; CHECK: uabalt z0.d, z3.s, z4.s
; CHECK: uabalt z1.d, z3.s, z4.s
; CHECK: uabalt z2.d, z3.s, z4.s
; CHECK: uabalt z0.d, z3.s, z4.s
; CHECK: uabalt z1.d, z3.s, z4.s
; CHECK: uabalt z2.d, z3.s, z4.s
; CHECK: uabalt z0.d, z3.s, z4.s
; CHECK: uabalt z1.d, z3.s, z4.s
; CHECK: uabalt z2.d, z3.s, z4.s
; CHECK: uabalt z0.d, z3.s, z4.s
; CHECK: add z0.d, z2.d, z0.d
; CHECK: uabalt z1.d, z3.s, z4.s
; CHECK: add z0.d, z0.d, z1.d
; CHECK: uaddv d0, p0, z0.d
entry:
br label %loop
loop:
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
%acc_phi = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
%ptr1_i = getelementptr <vscale x 4 x i32>, ptr %ptr1, i32 %i
%ptr2_i = getelementptr <vscale x 4 x i32>, ptr %ptr2, i32 %i
%a = load <vscale x 4 x i32>, ptr %ptr1_i, align 1
%b = load <vscale x 4 x i32>, ptr %ptr2_i, align 1
%acc_next = call <vscale x 2 x i64> @llvm.aarch64.sve.uabalt.nxv2i64(<vscale x 2 x i64> %acc_phi,
<vscale x 4 x i32> %a,
<vscale x 4 x i32> %b)
%next_i = add i32 %i, 1
%cmp = icmp slt i32 %next_i, 16
br i1 %cmp, label %loop, label %exit
exit:
%reduce = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %acc_next)
ret i64 %reduce
}
declare <vscale x 2 x i64> @llvm.aarch64.sve.uabalt.nxv2i64(<vscale x 2 x i64>, <vscale x 4 x i32>, <vscale x 4 x i32>)
define i32 @uabalt_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {
; CHECK-LABEL: uabalt_i16_to_i32_accumulation:
; CHECK: uabdlt z1.s, z1.h, z2.h
; CHECK: uabalt z0.s, z3.h, z4.h
; CHECK: uabalt z1.s, z4.h, z2.h
; CHECK: uabdlt z2.s, z3.h, z2.h
; CHECK: uabalt z2.s, z4.h, z3.h
; CHECK: uabalt z0.s, z3.h, z4.h
; CHECK: uabalt z1.s, z3.h, z4.h
; CHECK: uabalt z2.s, z3.h, z4.h
; CHECK: uabalt z0.s, z3.h, z4.h
; CHECK: uabalt z1.s, z3.h, z4.h
; CHECK: uabalt z2.s, z3.h, z4.h
; CHECK: uabalt z0.s, z3.h, z4.h
; CHECK: uabalt z1.s, z3.h, z4.h
; CHECK: uabalt z2.s, z3.h, z4.h
; CHECK: uabalt z0.s, z3.h, z4.h
; CHECK: add z0.s, z2.s, z0.s
; CHECK: uabalt z1.s, z3.h, z4.h
; CHECK: add z0.s, z0.s, z1.s
; CHECK: uaddv d0, p0, z0.s
entry:
br label %loop
loop:
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
%acc_phi = phi <vscale x 4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
%ptr1_i = getelementptr <vscale x 8 x i16>, ptr %ptr1, i32 %i
%ptr2_i = getelementptr <vscale x 8 x i16>, ptr %ptr2, i32 %i
%a = load <vscale x 8 x i16>, ptr %ptr1_i, align 1
%b = load <vscale x 8 x i16>, ptr %ptr2_i, align 1
%acc_next = call <vscale x 4 x i32> @llvm.aarch64.sve.uabalt.nxv4i32(<vscale x 4 x i32> %acc_phi,
<vscale x 8 x i16> %a,
<vscale x 8 x i16> %b)
%next_i = add i32 %i, 1
%cmp = icmp slt i32 %next_i, 16
br i1 %cmp, label %loop, label %exit
exit:
%reduce = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %acc_next)
ret i32 %reduce
}
declare <vscale x 4 x i32> @llvm.aarch64.sve.uabalt.nxv4i32(<vscale x 4 x i32>, <vscale x 8 x i16>, <vscale x 8 x i16>)
define i16 @uabalt_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {
; CHECK-LABEL: uabalt_i8_to_i16_accumulation:
; CHECK: uabdlt z1.h, z1.b, z2.b
; CHECK: uabalt z0.h, z3.b, z4.b
; CHECK: uabalt z1.h, z4.b, z2.b
; CHECK: uabdlt z2.h, z3.b, z2.b
; CHECK: uabalt z2.h, z4.b, z3.b
; CHECK: uabalt z0.h, z3.b, z4.b
; CHECK: uabalt z1.h, z3.b, z4.b
; CHECK: uabalt z2.h, z3.b, z4.b
; CHECK: uabalt z0.h, z3.b, z4.b
; CHECK: uabalt z1.h, z3.b, z4.b
; CHECK: uabalt z2.h, z3.b, z4.b
; CHECK: uabalt z0.h, z3.b, z4.b
; CHECK: uabalt z1.h, z3.b, z4.b
; CHECK: uabalt z2.h, z3.b, z4.b
; CHECK: uabalt z0.h, z3.b, z4.b
; CHECK: add z0.h, z2.h, z0.h
; CHECK: uabalt z1.h, z3.b, z4.b
; CHECK: add z0.h, z0.h, z1.h
; CHECK: uaddv d0, p0, z0.h
entry:
br label %loop
loop:
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
%acc_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]
%ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i
%ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i
%a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1
%b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1
%acc_next = call <vscale x 8 x i16> @llvm.aarch64.sve.uabalt.nxv8i16(<vscale x 8 x i16> %acc_phi,
<vscale x 16 x i8> %a,
<vscale x 16 x i8> %b)
%next_i = add i32 %i, 1
%cmp = icmp slt i32 %next_i, 16
br i1 %cmp, label %loop, label %exit
exit:
%reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)
ret i16 %reduce
}
declare <vscale x 8 x i16> @llvm.aarch64.sve.uabalt.nxv8i16(<vscale x 8 x i16>, <vscale x 16 x i8>, <vscale x 16 x i8>)
define i16 @uabalt_and_uabalb_accumulation(ptr %ptr1, ptr %ptr2) {
; CHECK-LABEL: uabalt_and_uabalb_accumulation:
; CHECK: uabalb z1.h, z2.b, z3.b
; CHECK: uabalt z0.h, z2.b, z3.b
; CHECK: uabalb z1.h, z6.b, z7.b
; CHECK: uabalt z0.h, z6.b, z7.b
; CHECK: uabdlb z7.h, z4.b, z5.b
; CHECK: uabdlt z4.h, z4.b, z5.b
; CHECK: uabalb z7.h, z6.b, z5.b
; CHECK: uabalt z4.h, z6.b, z5.b
; CHECK: uabdlb z6.h, z2.b, z3.b
; CHECK: uabdlt z2.h, z2.b, z3.b
; CHECK: uabalb z6.h, z5.b, z3.b
; CHECK: uabalt z2.h, z5.b, z3.b
; CHECK: uabalb z1.h, z3.b, z5.b
; CHECK: uabalt z0.h, z3.b, z5.b
; CHECK: uabalb z7.h, z3.b, z5.b
; CHECK: uabalt z4.h, z3.b, z5.b
; CHECK: uabalb z6.h, z3.b, z5.b
; CHECK: uabalt z2.h, z3.b, z5.b
; CHECK: uabalb z1.h, z3.b, z5.b
; CHECK: uabalt z0.h, z3.b, z5.b
; CHECK: uabalb z7.h, z3.b, z5.b
; CHECK: uabalt z4.h, z3.b, z5.b
; CHECK: uabalb z6.h, z3.b, z5.b
; CHECK: uabalt z2.h, z3.b, z5.b
; CHECK: uabalb z1.h, z3.b, z5.b
; CHECK: uabalt z0.h, z3.b, z5.b
; CHECK: uabalb z7.h, z3.b, z5.b
; CHECK: uabalt z4.h, z3.b, z5.b
; CHECK: uabalb z6.h, z3.b, z5.b
; CHECK: uabalt z2.h, z3.b, z5.b
; CHECK: add z2.h, z4.h, z2.h
; CHECK: uabalb z1.h, z3.b, z5.b
; CHECK: uabalt z0.h, z3.b, z5.b
; CHECK: add z3.h, z7.h, z6.h
; CHECK: add z1.h, z3.h, z1.h
; CHECK: add z0.h, z2.h, z0.h
; CHECK: add z1.h, p0/m, z1.h, z0.h
; CHECK: uaddv d0, p0, z1.h
entry:
br label %loop
loop:
%i = phi i32 [ 0, %entry ], [ %next_i, %loop ]
%acc_hi_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next_hi, %loop ]
%acc_lo_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next_lo, %loop ]
%ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i
%ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i
%a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1
%b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1
%acc_next_lo = call <vscale x 8 x i16> @llvm.aarch64.sve.uabalb.nxv8i16(<vscale x 8 x i16> %acc_lo_phi,
<vscale x 16 x i8> %a,
<vscale x 16 x i8> %b)
%acc_next_hi = call <vscale x 8 x i16> @llvm.aarch64.sve.uabalt.nxv8i16(<vscale x 8 x i16> %acc_hi_phi,
<vscale x 16 x i8> %a,
<vscale x 16 x i8> %b)
%next_i = add i32 %i, 1
%cmp = icmp slt i32 %next_i, 16
br i1 %cmp, label %loop, label %exit
exit:
%mask = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
%acc_next = tail call <vscale x 8 x i16> @llvm.aarch64.sve.add.nxv8i16(<vscale x 8 x i1> %mask, <vscale x 8 x i16> %acc_next_lo, <vscale x 8 x i16> %acc_next_hi)
%reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)
ret i16 %reduce
}
declare <vscale x 8 x i16> @llvm.aarch64.sve.add.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)