; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck -check-prefixes=CHECK,DAG %s
; RUN: llc < %s -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* -mtriple=arm64-eabi -aarch64-neon-syntax=apple 2>&1 | FileCheck %s --check-prefixes=CHECK,GISEL,FALLBACK

; FALLBACK-NOT: remark:{{.*}} sabdl8h
define <8 x i16> @sabdl8h(<8 x i8>* %A, <8 x i8>* %B) nounwind {
; CHECK-LABEL: sabdl8h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    sabdl.8h v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <8 x i8>, <8 x i8>* %A
  %tmp2 = load <8 x i8>, <8 x i8>* %B
  %tmp3 = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
  %tmp4 = zext <8 x i8> %tmp3 to <8 x i16>
  ret <8 x i16> %tmp4
}

; FALLBACK-NOT: remark:{{.*}} sabdl4s
define <4 x i32> @sabdl4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
; CHECK-LABEL: sabdl4s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    sabdl.4s v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <4 x i16>, <4 x i16>* %A
  %tmp2 = load <4 x i16>, <4 x i16>* %B
  %tmp3 = call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
  %tmp4 = zext <4 x i16> %tmp3 to <4 x i32>
  ret <4 x i32> %tmp4
}

; FALLBACK-NOT: remark:{{.*}} sabdl2d
define <2 x i64> @sabdl2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
; CHECK-LABEL: sabdl2d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    sabdl.2d v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <2 x i32>, <2 x i32>* %A
  %tmp2 = load <2 x i32>, <2 x i32>* %B
  %tmp3 = call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
  %tmp4 = zext <2 x i32> %tmp3 to <2 x i64>
  ret <2 x i64> %tmp4
}

define <8 x i16> @sabdl2_8h(<16 x i8>* %A, <16 x i8>* %B) nounwind {
; CHECK-LABEL: sabdl2_8h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0, #8]
; CHECK-NEXT:    ldr d1, [x1, #8]
; CHECK-NEXT:    sabdl.8h v0, v0, v1
; CHECK-NEXT:    ret
  %load1 = load <16 x i8>, <16 x i8>* %A
  %load2 = load <16 x i8>, <16 x i8>* %B
  %tmp1 = shufflevector <16 x i8> %load1, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  %tmp2 = shufflevector <16 x i8> %load2, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  %tmp3 = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
  %tmp4 = zext <8 x i8> %tmp3 to <8 x i16>
  ret <8 x i16> %tmp4
}

define <4 x i32> @sabdl2_4s(<8 x i16>* %A, <8 x i16>* %B) nounwind {
; CHECK-LABEL: sabdl2_4s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0, #8]
; CHECK-NEXT:    ldr d1, [x1, #8]
; CHECK-NEXT:    sabdl.4s v0, v0, v1
; CHECK-NEXT:    ret
  %load1 = load <8 x i16>, <8 x i16>* %A
  %load2 = load <8 x i16>, <8 x i16>* %B
  %tmp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
  %tmp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
  %tmp3 = call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
  %tmp4 = zext <4 x i16> %tmp3 to <4 x i32>
  ret <4 x i32> %tmp4
}

define <2 x i64> @sabdl2_2d(<4 x i32>* %A, <4 x i32>* %B) nounwind {
; CHECK-LABEL: sabdl2_2d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0, #8]
; CHECK-NEXT:    ldr d1, [x1, #8]
; CHECK-NEXT:    sabdl.2d v0, v0, v1
; CHECK-NEXT:    ret
  %load1 = load <4 x i32>, <4 x i32>* %A
  %load2 = load <4 x i32>, <4 x i32>* %B
  %tmp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
  %tmp2 = shufflevector <4 x i32> %load2, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
  %tmp3 = call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
  %tmp4 = zext <2 x i32> %tmp3 to <2 x i64>
  ret <2 x i64> %tmp4
}

; FALLBACK-NOT: remark:{{.*}} uabdl8h)
define <8 x i16> @uabdl8h(<8 x i8>* %A, <8 x i8>* %B) nounwind {
; CHECK-LABEL: uabdl8h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    uabdl.8h v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <8 x i8>, <8 x i8>* %A
  %tmp2 = load <8 x i8>, <8 x i8>* %B
  %tmp3 = call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
  %tmp4 = zext <8 x i8> %tmp3 to <8 x i16>
  ret <8 x i16> %tmp4
}

; FALLBACK-NOT: remark:{{.*}} uabdl4s)
define <4 x i32> @uabdl4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
; CHECK-LABEL: uabdl4s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    uabdl.4s v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <4 x i16>, <4 x i16>* %A
  %tmp2 = load <4 x i16>, <4 x i16>* %B
  %tmp3 = call <4 x i16> @llvm.aarch64.neon.uabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
  %tmp4 = zext <4 x i16> %tmp3 to <4 x i32>
  ret <4 x i32> %tmp4
}

; FALLBACK-NOT: remark:{{.*}} uabdl2d)
define <2 x i64> @uabdl2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
; CHECK-LABEL: uabdl2d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    uabdl.2d v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <2 x i32>, <2 x i32>* %A
  %tmp2 = load <2 x i32>, <2 x i32>* %B
  %tmp3 = call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
  %tmp4 = zext <2 x i32> %tmp3 to <2 x i64>
  ret <2 x i64> %tmp4
}

define <8 x i16> @uabdl2_8h(<16 x i8>* %A, <16 x i8>* %B) nounwind {
; CHECK-LABEL: uabdl2_8h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0, #8]
; CHECK-NEXT:    ldr d1, [x1, #8]
; CHECK-NEXT:    uabdl.8h v0, v0, v1
; CHECK-NEXT:    ret
  %load1 = load <16 x i8>, <16 x i8>* %A
  %load2 = load <16 x i8>, <16 x i8>* %B
  %tmp1 = shufflevector <16 x i8> %load1, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  %tmp2 = shufflevector <16 x i8> %load2, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>

  %tmp3 = call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
  %tmp4 = zext <8 x i8> %tmp3 to <8 x i16>
  ret <8 x i16> %tmp4
}

define <4 x i32> @uabdl2_4s(<8 x i16>* %A, <8 x i16>* %B) nounwind {
; CHECK-LABEL: uabdl2_4s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0, #8]
; CHECK-NEXT:    ldr d1, [x1, #8]
; CHECK-NEXT:    uabdl.4s v0, v0, v1
; CHECK-NEXT:    ret
  %load1 = load <8 x i16>, <8 x i16>* %A
  %load2 = load <8 x i16>, <8 x i16>* %B
  %tmp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
  %tmp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
  %tmp3 = call <4 x i16> @llvm.aarch64.neon.uabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
  %tmp4 = zext <4 x i16> %tmp3 to <4 x i32>
  ret <4 x i32> %tmp4
}

define <2 x i64> @uabdl2_2d(<4 x i32>* %A, <4 x i32>* %B) nounwind {
; CHECK-LABEL: uabdl2_2d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0, #8]
; CHECK-NEXT:    ldr d1, [x1, #8]
; CHECK-NEXT:    uabdl.2d v0, v0, v1
; CHECK-NEXT:    ret
  %load1 = load <4 x i32>, <4 x i32>* %A
  %load2 = load <4 x i32>, <4 x i32>* %B
  %tmp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
  %tmp2 = shufflevector <4 x i32> %load2, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
  %tmp3 = call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
  %tmp4 = zext <2 x i32> %tmp3 to <2 x i64>
  ret <2 x i64> %tmp4
}

declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)
declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)

define i16 @uabd16b_rdx(<16 x i8>* %a, <16 x i8>* %b) {
; CHECK-LABEL: uabd16b_rdx:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    ldr q1, [x1]
; CHECK-NEXT:    uabd.16b v0, v0, v1
; CHECK-NEXT:    ushll.8h v1, v0, #0
; CHECK-NEXT:    uaddw2.8h v0, v1, v0
; CHECK-NEXT:    addv.8h h0, v0
; CHECK-NEXT:    fmov w0, s0
; CHECK-NEXT:    ret
  %aload = load <16 x i8>, <16 x i8>* %a, align 1
  %bload = load <16 x i8>, <16 x i8>* %b, align 1
  %aext = zext <16 x i8> %aload to <16 x i16>
  %bext = zext <16 x i8> %bload to <16 x i16>
  %abdiff = sub nsw <16 x i16> %aext, %bext
  %abcmp = icmp slt <16 x i16> %abdiff, zeroinitializer
  %ababs = sub nsw <16 x i16> zeroinitializer, %abdiff
  %absel = select <16 x i1> %abcmp, <16 x i16> %ababs, <16 x i16> %abdiff
  %reduced_v = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %absel)
  ret i16 %reduced_v
}

define i32 @uabd16b_rdx_i32(<16 x i8> %a, <16 x i8> %b) {
; CHECK-LABEL: uabd16b_rdx_i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    uabdl.8h v2, v0, v1
; CHECK-NEXT:    uabal2.8h v2, v0, v1
; CHECK-NEXT:    uaddlv.8h s0, v2
; CHECK-NEXT:    fmov w0, s0
; CHECK-NEXT:    ret
  %aext = zext <16 x i8> %a to <16 x i32>
  %bext = zext <16 x i8> %b to <16 x i32>
  %abdiff = sub nsw <16 x i32> %aext, %bext
  %abcmp = icmp slt <16 x i32> %abdiff, zeroinitializer
  %ababs = sub nsw <16 x i32> zeroinitializer, %abdiff
  %absel = select <16 x i1> %abcmp, <16 x i32> %ababs, <16 x i32> %abdiff
  %reduced_v = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %absel)
  ret i32 %reduced_v
}

define i32 @sabd16b_rdx_i32(<16 x i8> %a, <16 x i8> %b) {
; CHECK-LABEL: sabd16b_rdx_i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sabdl.8h v2, v0, v1
; CHECK-NEXT:    sabal2.8h v2, v0, v1
; CHECK-NEXT:    uaddlv.8h s0, v2
; CHECK-NEXT:    fmov w0, s0
; CHECK-NEXT:    ret
  %aext = sext <16 x i8> %a to <16 x i32>
  %bext = sext <16 x i8> %b to <16 x i32>
  %abdiff = sub nsw <16 x i32> %aext, %bext
  %abcmp = icmp slt <16 x i32> %abdiff, zeroinitializer
  %ababs = sub nsw <16 x i32> zeroinitializer, %abdiff
  %absel = select <16 x i1> %abcmp, <16 x i32> %ababs, <16 x i32> %abdiff
  %reduced_v = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %absel)
  ret i32 %reduced_v
}


declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)

define i32 @uabd8h_rdx(<8 x i16>* %a, <8 x i16>* %b) {
; CHECK-LABEL: uabd8h_rdx:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    ldr q1, [x1]
; CHECK-NEXT:    uabd.8h v0, v0, v1
; CHECK-NEXT:    ushll.4s v1, v0, #0
; CHECK-NEXT:    uaddw2.4s v0, v1, v0
; CHECK-NEXT:    addv.4s s0, v0
; CHECK-NEXT:    fmov w0, s0
; CHECK-NEXT:    ret
  %aload = load <8 x i16>, <8 x i16>* %a, align 1
  %bload = load <8 x i16>, <8 x i16>* %b, align 1
  %aext = zext <8 x i16> %aload to <8 x i32>
  %bext = zext <8 x i16> %bload to <8 x i32>
  %abdiff = sub nsw <8 x i32> %aext, %bext
  %abcmp = icmp slt <8 x i32> %abdiff, zeroinitializer
  %ababs = sub nsw <8 x i32> zeroinitializer, %abdiff
  %absel = select <8 x i1> %abcmp, <8 x i32> %ababs, <8 x i32> %abdiff
  %reduced_v = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %absel)
  ret i32 %reduced_v
}

define i32 @sabd8h_rdx(<8 x i16> %a, <8 x i16> %b) {
; CHECK-LABEL: sabd8h_rdx:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sabd.8h v0, v0, v1
; CHECK-NEXT:    ushll.4s v1, v0, #0
; CHECK-NEXT:    uaddw2.4s v0, v1, v0
; CHECK-NEXT:    addv.4s s0, v0
; CHECK-NEXT:    fmov w0, s0
; CHECK-NEXT:    ret
  %aext = sext <8 x i16> %a to <8 x i32>
  %bext = sext <8 x i16> %b to <8 x i32>
  %abdiff = sub nsw <8 x i32> %aext, %bext
  %abcmp = icmp slt <8 x i32> %abdiff, zeroinitializer
  %ababs = sub nsw <8 x i32> zeroinitializer, %abdiff
  %absel = select <8 x i1> %abcmp, <8 x i32> %ababs, <8 x i32> %abdiff
  %reduced_v = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %absel)
  ret i32 %reduced_v
}

define i32 @uabdl4s_rdx_i32(<4 x i16> %a, <4 x i16> %b) {
; DAG-LABEL: uabdl4s_rdx_i32:
; DAG:       // %bb.0:
; DAG-NEXT:    uabdl.4s v0, v0, v1
; DAG-NEXT:    addv.4s s0, v0
; DAG-NEXT:    fmov w0, s0
; DAG-NEXT:    ret
;
; GISEL-LABEL: uabdl4s_rdx_i32:
; GISEL:       // %bb.0:
; GISEL-NEXT:    movi.2d v2, #0000000000000000
; GISEL-NEXT:    usubl.4s v0, v0, v1
; GISEL-NEXT:    cmgt.4s v1, v2, v0
; GISEL-NEXT:    neg.4s v2, v0
; GISEL-NEXT:    shl.4s v1, v1, #31
; GISEL-NEXT:    sshr.4s v1, v1, #31
; GISEL-NEXT:    bit.16b v0, v2, v1
; GISEL-NEXT:    addv.4s s0, v0
; GISEL-NEXT:    fmov w0, s0
; GISEL-NEXT:    ret

; GISel doesn't match this pattern yet.
  %aext = zext <4 x i16> %a to <4 x i32>
  %bext = zext <4 x i16> %b to <4 x i32>
 %abdiff = sub nsw <4 x i32> %aext, %bext
  %abcmp = icmp slt <4 x i32> %abdiff, zeroinitializer
  %ababs = sub nsw <4 x i32> zeroinitializer, %abdiff
  %absel = select <4 x i1> %abcmp, <4 x i32> %ababs, <4 x i32> %abdiff
  %reduced_v = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %absel)
  ret i32 %reduced_v
}

declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)
declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)

define i64 @uabd4s_rdx(<4 x i32>* %a, <4 x i32>* %b, i32 %h) {
; CHECK-LABEL: uabd4s_rdx:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    ldr q1, [x1]
; CHECK-NEXT:    uabd.4s v0, v0, v1
; CHECK-NEXT:    ushll.2d v1, v0, #0
; CHECK-NEXT:    uaddw2.2d v0, v1, v0
; CHECK-NEXT:    addp.2d d0, v0
; CHECK-NEXT:    fmov x0, d0
; CHECK-NEXT:    ret
  %aload = load <4 x i32>, <4 x i32>* %a, align 1
  %bload = load <4 x i32>, <4 x i32>* %b, align 1
  %aext = zext <4 x i32> %aload to <4 x i64>
  %bext = zext <4 x i32> %bload to <4 x i64>
  %abdiff = sub nsw <4 x i64> %aext, %bext
  %abcmp = icmp slt <4 x i64> %abdiff, zeroinitializer
  %ababs = sub nsw <4 x i64> zeroinitializer, %abdiff
  %absel = select <4 x i1> %abcmp, <4 x i64> %ababs, <4 x i64> %abdiff
  %reduced_v = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %absel)
  ret i64 %reduced_v
}

define i64 @sabd4s_rdx(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: sabd4s_rdx:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sabd.4s v0, v0, v1
; CHECK-NEXT:    ushll.2d v1, v0, #0
; CHECK-NEXT:    uaddw2.2d v0, v1, v0
; CHECK-NEXT:    addp.2d d0, v0
; CHECK-NEXT:    fmov x0, d0
; CHECK-NEXT:    ret
  %aext = sext <4 x i32> %a to <4 x i64>
  %bext = sext <4 x i32> %b to <4 x i64>
  %abdiff = sub nsw <4 x i64> %aext, %bext
  %abcmp = icmp slt <4 x i64> %abdiff, zeroinitializer
  %ababs = sub nsw <4 x i64> zeroinitializer, %abdiff
  %absel = select <4 x i1> %abcmp, <4 x i64> %ababs, <4 x i64> %abdiff
  %reduced_v = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %absel)
  ret i64 %reduced_v
}

define i64 @uabdl2d_rdx_i64(<2 x i32> %a, <2 x i32> %b) {
; DAG-LABEL: uabdl2d_rdx_i64:
; DAG:       // %bb.0:
; DAG-NEXT:    uabdl.2d v0, v0, v1
; DAG-NEXT:    addp.2d d0, v0
; DAG-NEXT:    fmov x0, d0
; DAG-NEXT:    ret
;
; GISEL-LABEL: uabdl2d_rdx_i64:
; GISEL:       // %bb.0:
; GISEL-NEXT:    movi.2d v2, #0000000000000000
; GISEL-NEXT:    usubl.2d v0, v0, v1
; GISEL-NEXT:    cmgt.2d v1, v2, v0
; GISEL-NEXT:    neg.2d v2, v0
; GISEL-NEXT:    shl.2d v1, v1, #63
; GISEL-NEXT:    sshr.2d v1, v1, #63
; GISEL-NEXT:    bit.16b v0, v2, v1
; GISEL-NEXT:    addp.2d d0, v0
; GISEL-NEXT:    fmov x0, d0
; GISEL-NEXT:    ret

; GISel doesn't match this pattern yet
  %aext = zext <2 x i32> %a to <2 x i64>
  %bext = zext <2 x i32> %b to <2 x i64>
  %abdiff = sub nsw <2 x i64> %aext, %bext
  %abcmp = icmp slt <2 x i64> %abdiff, zeroinitializer
  %ababs = sub nsw <2 x i64> zeroinitializer, %abdiff
  %absel = select <2 x i1> %abcmp, <2 x i64> %ababs, <2 x i64> %abdiff
  %reduced_v = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %absel)
  ret i64 %reduced_v
}

define <2 x float> @fabd_2s(<2 x float>* %A, <2 x float>* %B) nounwind {
; CHECK-LABEL: fabd_2s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    fabd.2s v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <2 x float>, <2 x float>* %A
  %tmp2 = load <2 x float>, <2 x float>* %B
  %tmp3 = call <2 x float> @llvm.aarch64.neon.fabd.v2f32(<2 x float> %tmp1, <2 x float> %tmp2)
  ret <2 x float> %tmp3
}

define <4 x float> @fabd_4s(<4 x float>* %A, <4 x float>* %B) nounwind {
; CHECK-LABEL: fabd_4s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    ldr q1, [x1]
; CHECK-NEXT:    fabd.4s v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <4 x float>, <4 x float>* %A
  %tmp2 = load <4 x float>, <4 x float>* %B
  %tmp3 = call <4 x float> @llvm.aarch64.neon.fabd.v4f32(<4 x float> %tmp1, <4 x float> %tmp2)
  ret <4 x float> %tmp3
}

define <2 x double> @fabd_2d(<2 x double>* %A, <2 x double>* %B) nounwind {
; CHECK-LABEL: fabd_2d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    ldr q1, [x1]
; CHECK-NEXT:    fabd.2d v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <2 x double>, <2 x double>* %A
  %tmp2 = load <2 x double>, <2 x double>* %B
  %tmp3 = call <2 x double> @llvm.aarch64.neon.fabd.v2f64(<2 x double> %tmp1, <2 x double> %tmp2)
  ret <2 x double> %tmp3
}

declare <2 x float> @llvm.aarch64.neon.fabd.v2f32(<2 x float>, <2 x float>) nounwind readnone
declare <4 x float> @llvm.aarch64.neon.fabd.v4f32(<4 x float>, <4 x float>) nounwind readnone
declare <2 x double> @llvm.aarch64.neon.fabd.v2f64(<2 x double>, <2 x double>) nounwind readnone

define <2 x float> @fabd_2s_from_fsub_fabs(<2 x float>* %A, <2 x float>* %B) nounwind {
; CHECK-LABEL: fabd_2s_from_fsub_fabs:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    fabd.2s v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <2 x float>, <2 x float>* %A
  %tmp2 = load <2 x float>, <2 x float>* %B
  %sub = fsub <2 x float> %tmp1, %tmp2
  %abs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %sub)
  ret <2 x float> %abs
}

define <4 x float> @fabd_4s_from_fsub_fabs(<4 x float>* %A, <4 x float>* %B) nounwind {
; CHECK-LABEL: fabd_4s_from_fsub_fabs:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    ldr q1, [x1]
; CHECK-NEXT:    fabd.4s v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <4 x float>, <4 x float>* %A
  %tmp2 = load <4 x float>, <4 x float>* %B
  %sub = fsub <4 x float> %tmp1, %tmp2
  %abs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %sub)
  ret <4 x float> %abs
}

define <2 x double> @fabd_2d_from_fsub_fabs(<2 x double>* %A, <2 x double>* %B) nounwind {
; CHECK-LABEL: fabd_2d_from_fsub_fabs:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    ldr q1, [x1]
; CHECK-NEXT:    fabd.2d v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <2 x double>, <2 x double>* %A
  %tmp2 = load <2 x double>, <2 x double>* %B
  %sub = fsub <2 x double> %tmp1, %tmp2
  %abs = call <2 x double> @llvm.fabs.v2f64(<2 x double> %sub)
  ret <2 x double> %abs
}

declare <2 x float> @llvm.fabs.v2f32(<2 x float>) nounwind readnone
declare <4 x float> @llvm.fabs.v4f32(<4 x float>) nounwind readnone
declare <2 x double> @llvm.fabs.v2f64(<2 x double>) nounwind readnone

define <8 x i8> @sabd_8b(<8 x i8>* %A, <8 x i8>* %B) nounwind {
; CHECK-LABEL: sabd_8b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    sabd.8b v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <8 x i8>, <8 x i8>* %A
  %tmp2 = load <8 x i8>, <8 x i8>* %B
  %tmp3 = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
  ret <8 x i8> %tmp3
}

define <16 x i8> @sabd_16b(<16 x i8>* %A, <16 x i8>* %B) nounwind {
; CHECK-LABEL: sabd_16b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    ldr q1, [x1]
; CHECK-NEXT:    sabd.16b v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <16 x i8>, <16 x i8>* %A
  %tmp2 = load <16 x i8>, <16 x i8>* %B
  %tmp3 = call <16 x i8> @llvm.aarch64.neon.sabd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
  ret <16 x i8> %tmp3
}

define <4 x i16> @sabd_4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
; CHECK-LABEL: sabd_4h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    sabd.4h v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <4 x i16>, <4 x i16>* %A
  %tmp2 = load <4 x i16>, <4 x i16>* %B
  %tmp3 = call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
  ret <4 x i16> %tmp3
}

define <8 x i16> @sabd_8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
; CHECK-LABEL: sabd_8h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    ldr q1, [x1]
; CHECK-NEXT:    sabd.8h v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <8 x i16>, <8 x i16>* %A
  %tmp2 = load <8 x i16>, <8 x i16>* %B
  %tmp3 = call <8 x i16> @llvm.aarch64.neon.sabd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
  ret <8 x i16> %tmp3
}

define <2 x i32> @sabd_2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
; CHECK-LABEL: sabd_2s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    sabd.2s v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <2 x i32>, <2 x i32>* %A
  %tmp2 = load <2 x i32>, <2 x i32>* %B
  %tmp3 = call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
  ret <2 x i32> %tmp3
}

define <4 x i32> @sabd_4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
; CHECK-LABEL: sabd_4s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    ldr q1, [x1]
; CHECK-NEXT:    sabd.4s v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <4 x i32>, <4 x i32>* %A
  %tmp2 = load <4 x i32>, <4 x i32>* %B
  %tmp3 = call <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
  ret <4 x i32> %tmp3
}

declare <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
declare <16 x i8> @llvm.aarch64.neon.sabd.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
declare <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
declare <8 x i16> @llvm.aarch64.neon.sabd.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
declare <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
declare <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32>, <4 x i32>) nounwind readnone

define <8 x i8> @uabd_8b(<8 x i8>* %A, <8 x i8>* %B) nounwind {
; CHECK-LABEL: uabd_8b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    uabd.8b v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <8 x i8>, <8 x i8>* %A
  %tmp2 = load <8 x i8>, <8 x i8>* %B
  %tmp3 = call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
  ret <8 x i8> %tmp3
}

define <16 x i8> @uabd_16b(<16 x i8>* %A, <16 x i8>* %B) nounwind {
; CHECK-LABEL: uabd_16b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    ldr q1, [x1]
; CHECK-NEXT:    uabd.16b v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <16 x i8>, <16 x i8>* %A
  %tmp2 = load <16 x i8>, <16 x i8>* %B
  %tmp3 = call <16 x i8> @llvm.aarch64.neon.uabd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
  ret <16 x i8> %tmp3
}

define <4 x i16> @uabd_4h(<4 x i16>* %A, <4 x i16>* %B) nounwind {
; CHECK-LABEL: uabd_4h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    uabd.4h v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <4 x i16>, <4 x i16>* %A
  %tmp2 = load <4 x i16>, <4 x i16>* %B
  %tmp3 = call <4 x i16> @llvm.aarch64.neon.uabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
  ret <4 x i16> %tmp3
}

define <8 x i16> @uabd_8h(<8 x i16>* %A, <8 x i16>* %B) nounwind {
; CHECK-LABEL: uabd_8h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    ldr q1, [x1]
; CHECK-NEXT:    uabd.8h v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <8 x i16>, <8 x i16>* %A
  %tmp2 = load <8 x i16>, <8 x i16>* %B
  %tmp3 = call <8 x i16> @llvm.aarch64.neon.uabd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
  ret <8 x i16> %tmp3
}

define <2 x i32> @uabd_2s(<2 x i32>* %A, <2 x i32>* %B) nounwind {
; CHECK-LABEL: uabd_2s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    ldr d1, [x1]
; CHECK-NEXT:    uabd.2s v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <2 x i32>, <2 x i32>* %A
  %tmp2 = load <2 x i32>, <2 x i32>* %B
  %tmp3 = call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
  ret <2 x i32> %tmp3
}

define <4 x i32> @uabd_4s(<4 x i32>* %A, <4 x i32>* %B) nounwind {
; CHECK-LABEL: uabd_4s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    ldr q1, [x1]
; CHECK-NEXT:    uabd.4s v0, v0, v1
; CHECK-NEXT:    ret
  %tmp1 = load <4 x i32>, <4 x i32>* %A
  %tmp2 = load <4 x i32>, <4 x i32>* %B
  %tmp3 = call <4 x i32> @llvm.aarch64.neon.uabd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
  ret <4 x i32> %tmp3
}

declare <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
declare <16 x i8> @llvm.aarch64.neon.uabd.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
declare <4 x i16> @llvm.aarch64.neon.uabd.v4i16(<4 x i16>, <4 x i16>) nounwind readnone
declare <8 x i16> @llvm.aarch64.neon.uabd.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
declare <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
declare <4 x i32> @llvm.aarch64.neon.uabd.v4i32(<4 x i32>, <4 x i32>) nounwind readnone

define <8 x i8> @sqabs_8b(<8 x i8>* %A) nounwind {
; CHECK-LABEL: sqabs_8b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    sqabs.8b v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <8 x i8>, <8 x i8>* %A
  %tmp3 = call <8 x i8> @llvm.aarch64.neon.sqabs.v8i8(<8 x i8> %tmp1)
  ret <8 x i8> %tmp3
}

define <16 x i8> @sqabs_16b(<16 x i8>* %A) nounwind {
; CHECK-LABEL: sqabs_16b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    sqabs.16b v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <16 x i8>, <16 x i8>* %A
  %tmp3 = call <16 x i8> @llvm.aarch64.neon.sqabs.v16i8(<16 x i8> %tmp1)
  ret <16 x i8> %tmp3
}

define <4 x i16> @sqabs_4h(<4 x i16>* %A) nounwind {
; CHECK-LABEL: sqabs_4h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    sqabs.4h v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <4 x i16>, <4 x i16>* %A
  %tmp3 = call <4 x i16> @llvm.aarch64.neon.sqabs.v4i16(<4 x i16> %tmp1)
  ret <4 x i16> %tmp3
}

define <8 x i16> @sqabs_8h(<8 x i16>* %A) nounwind {
; CHECK-LABEL: sqabs_8h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    sqabs.8h v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <8 x i16>, <8 x i16>* %A
  %tmp3 = call <8 x i16> @llvm.aarch64.neon.sqabs.v8i16(<8 x i16> %tmp1)
  ret <8 x i16> %tmp3
}

define <2 x i32> @sqabs_2s(<2 x i32>* %A) nounwind {
; CHECK-LABEL: sqabs_2s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    sqabs.2s v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <2 x i32>, <2 x i32>* %A
  %tmp3 = call <2 x i32> @llvm.aarch64.neon.sqabs.v2i32(<2 x i32> %tmp1)
  ret <2 x i32> %tmp3
}

define <4 x i32> @sqabs_4s(<4 x i32>* %A) nounwind {
; CHECK-LABEL: sqabs_4s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    sqabs.4s v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <4 x i32>, <4 x i32>* %A
  %tmp3 = call <4 x i32> @llvm.aarch64.neon.sqabs.v4i32(<4 x i32> %tmp1)
  ret <4 x i32> %tmp3
}

declare <8 x i8> @llvm.aarch64.neon.sqabs.v8i8(<8 x i8>) nounwind readnone
declare <16 x i8> @llvm.aarch64.neon.sqabs.v16i8(<16 x i8>) nounwind readnone
declare <4 x i16> @llvm.aarch64.neon.sqabs.v4i16(<4 x i16>) nounwind readnone
declare <8 x i16> @llvm.aarch64.neon.sqabs.v8i16(<8 x i16>) nounwind readnone
declare <2 x i32> @llvm.aarch64.neon.sqabs.v2i32(<2 x i32>) nounwind readnone
declare <4 x i32> @llvm.aarch64.neon.sqabs.v4i32(<4 x i32>) nounwind readnone

define <8 x i8> @sqneg_8b(<8 x i8>* %A) nounwind {
; CHECK-LABEL: sqneg_8b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    sqneg.8b v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <8 x i8>, <8 x i8>* %A
  %tmp3 = call <8 x i8> @llvm.aarch64.neon.sqneg.v8i8(<8 x i8> %tmp1)
  ret <8 x i8> %tmp3
}

define <16 x i8> @sqneg_16b(<16 x i8>* %A) nounwind {
; CHECK-LABEL: sqneg_16b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    sqneg.16b v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <16 x i8>, <16 x i8>* %A
  %tmp3 = call <16 x i8> @llvm.aarch64.neon.sqneg.v16i8(<16 x i8> %tmp1)
  ret <16 x i8> %tmp3
}

define <4 x i16> @sqneg_4h(<4 x i16>* %A) nounwind {
; CHECK-LABEL: sqneg_4h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    sqneg.4h v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <4 x i16>, <4 x i16>* %A
  %tmp3 = call <4 x i16> @llvm.aarch64.neon.sqneg.v4i16(<4 x i16> %tmp1)
  ret <4 x i16> %tmp3
}

define <8 x i16> @sqneg_8h(<8 x i16>* %A) nounwind {
; CHECK-LABEL: sqneg_8h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    sqneg.8h v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <8 x i16>, <8 x i16>* %A
  %tmp3 = call <8 x i16> @llvm.aarch64.neon.sqneg.v8i16(<8 x i16> %tmp1)
  ret <8 x i16> %tmp3
}

define <2 x i32> @sqneg_2s(<2 x i32>* %A) nounwind {
; CHECK-LABEL: sqneg_2s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    sqneg.2s v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <2 x i32>, <2 x i32>* %A
  %tmp3 = call <2 x i32> @llvm.aarch64.neon.sqneg.v2i32(<2 x i32> %tmp1)
  ret <2 x i32> %tmp3
}

define <4 x i32> @sqneg_4s(<4 x i32>* %A) nounwind {
; CHECK-LABEL: sqneg_4s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    sqneg.4s v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <4 x i32>, <4 x i32>* %A
  %tmp3 = call <4 x i32> @llvm.aarch64.neon.sqneg.v4i32(<4 x i32> %tmp1)
  ret <4 x i32> %tmp3
}

declare <8 x i8> @llvm.aarch64.neon.sqneg.v8i8(<8 x i8>) nounwind readnone
declare <16 x i8> @llvm.aarch64.neon.sqneg.v16i8(<16 x i8>) nounwind readnone
declare <4 x i16> @llvm.aarch64.neon.sqneg.v4i16(<4 x i16>) nounwind readnone
declare <8 x i16> @llvm.aarch64.neon.sqneg.v8i16(<8 x i16>) nounwind readnone
declare <2 x i32> @llvm.aarch64.neon.sqneg.v2i32(<2 x i32>) nounwind readnone
declare <4 x i32> @llvm.aarch64.neon.sqneg.v4i32(<4 x i32>) nounwind readnone

define <8 x i8> @abs_8b(<8 x i8>* %A) nounwind {
; CHECK-LABEL: abs_8b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    abs.8b v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <8 x i8>, <8 x i8>* %A
  %tmp3 = call <8 x i8> @llvm.aarch64.neon.abs.v8i8(<8 x i8> %tmp1)
  ret <8 x i8> %tmp3
}

define <16 x i8> @abs_16b(<16 x i8>* %A) nounwind {
; CHECK-LABEL: abs_16b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    abs.16b v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <16 x i8>, <16 x i8>* %A
  %tmp3 = call <16 x i8> @llvm.aarch64.neon.abs.v16i8(<16 x i8> %tmp1)
  ret <16 x i8> %tmp3
}

define <4 x i16> @abs_4h(<4 x i16>* %A) nounwind {
; CHECK-LABEL: abs_4h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    abs.4h v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <4 x i16>, <4 x i16>* %A
  %tmp3 = call <4 x i16> @llvm.aarch64.neon.abs.v4i16(<4 x i16> %tmp1)
  ret <4 x i16> %tmp3
}

define <8 x i16> @abs_8h(<8 x i16>* %A) nounwind {
; CHECK-LABEL: abs_8h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    abs.8h v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <8 x i16>, <8 x i16>* %A
  %tmp3 = call <8 x i16> @llvm.aarch64.neon.abs.v8i16(<8 x i16> %tmp1)
  ret <8 x i16> %tmp3
}

define <2 x i32> @abs_2s(<2 x i32>* %A) nounwind {
; CHECK-LABEL: abs_2s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d0, [x0]
; CHECK-NEXT:    abs.2s v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <2 x i32>, <2 x i32>* %A
  %tmp3 = call <2 x i32> @llvm.aarch64.neon.abs.v2i32(<2 x i32> %tmp1)
  ret <2 x i32> %tmp3
}

define <4 x i32> @abs_4s(<4 x i32>* %A) nounwind {
; CHECK-LABEL: abs_4s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x0]
; CHECK-NEXT:    abs.4s v0, v0
; CHECK-NEXT:    ret
  %tmp1 = load <4 x i32>, <4 x i32>* %A
  %tmp3 = call <4 x i32> @llvm.aarch64.neon.abs.v4i32(<4 x i32> %tmp1)
  ret <4 x i32> %tmp3
}

define <1 x i64> @abs_1d(<1 x i64> %A) nounwind {
; CHECK-LABEL: abs_1d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    abs d0, d0
; CHECK-NEXT:    ret
  %abs = call <1 x i64> @llvm.aarch64.neon.abs.v1i64(<1 x i64> %A)
  ret <1 x i64> %abs
}

define i64 @abs_1d_honestly(i64 %A) nounwind {
; CHECK-LABEL: abs_1d_honestly:
; CHECK:       // %bb.0:
; CHECK-NEXT:    fmov d0, x0
; CHECK-NEXT:    abs d0, d0
; CHECK-NEXT:    fmov x0, d0
; CHECK-NEXT:    ret
  %abs = call i64 @llvm.aarch64.neon.abs.i64(i64 %A)
  ret i64 %abs
}

declare <8 x i8> @llvm.aarch64.neon.abs.v8i8(<8 x i8>) nounwind readnone
declare <16 x i8> @llvm.aarch64.neon.abs.v16i8(<16 x i8>) nounwind readnone
declare <4 x i16> @llvm.aarch64.neon.abs.v4i16(<4 x i16>) nounwind readnone
declare <8 x i16> @llvm.aarch64.neon.abs.v8i16(<8 x i16>) nounwind readnone
declare <2 x i32> @llvm.aarch64.neon.abs.v2i32(<2 x i32>) nounwind readnone
declare <4 x i32> @llvm.aarch64.neon.abs.v4i32(<4 x i32>) nounwind readnone
declare <1 x i64> @llvm.aarch64.neon.abs.v1i64(<1 x i64>) nounwind readnone
declare i64 @llvm.aarch64.neon.abs.i64(i64) nounwind readnone

; FALLBACK-NOT: remark:{{.*}} sabal8h
define <8 x i16> @sabal8h(<8 x i8>* %A, <8 x i8>* %B,  <8 x i16>* %C) nounwind {
; DAG-LABEL: sabal8h:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr d1, [x1]
; DAG-NEXT:    ldr d2, [x0]
; DAG-NEXT:    ldr q0, [x2]
; DAG-NEXT:    sabal.8h v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: sabal8h:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr d1, [x0]
; GISEL-NEXT:    ldr d2, [x1]
; GISEL-NEXT:    ldr q0, [x2]
; GISEL-NEXT:    sabal.8h v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <8 x i8>, <8 x i8>* %A
  %tmp2 = load <8 x i8>, <8 x i8>* %B
  %tmp3 = load <8 x i16>, <8 x i16>* %C
  %tmp4 = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
  %tmp4.1 = zext <8 x i8> %tmp4 to <8 x i16>
  %tmp5 = add <8 x i16> %tmp3, %tmp4.1
  ret <8 x i16> %tmp5
}

; FALLBACK-NOT: remark:{{.*}} sabal4s
define <4 x i32> @sabal4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
; DAG-LABEL: sabal4s:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr d1, [x1]
; DAG-NEXT:    ldr d2, [x0]
; DAG-NEXT:    ldr q0, [x2]
; DAG-NEXT:    sabal.4s v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: sabal4s:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr d1, [x0]
; GISEL-NEXT:    ldr d2, [x1]
; GISEL-NEXT:    ldr q0, [x2]
; GISEL-NEXT:    sabal.4s v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <4 x i16>, <4 x i16>* %A
  %tmp2 = load <4 x i16>, <4 x i16>* %B
  %tmp3 = load <4 x i32>, <4 x i32>* %C
  %tmp4 = call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
  %tmp4.1 = zext <4 x i16> %tmp4 to <4 x i32>
  %tmp5 = add <4 x i32> %tmp3, %tmp4.1
  ret <4 x i32> %tmp5
}

; FALLBACK-NOT: remark:{{.*}} sabal2d
define <2 x i64> @sabal2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
; DAG-LABEL: sabal2d:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr d1, [x1]
; DAG-NEXT:    ldr d2, [x0]
; DAG-NEXT:    ldr q0, [x2]
; DAG-NEXT:    sabal.2d v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: sabal2d:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr d1, [x0]
; GISEL-NEXT:    ldr d2, [x1]
; GISEL-NEXT:    ldr q0, [x2]
; GISEL-NEXT:    sabal.2d v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <2 x i32>, <2 x i32>* %A
  %tmp2 = load <2 x i32>, <2 x i32>* %B
  %tmp3 = load <2 x i64>, <2 x i64>* %C
  %tmp4 = call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
  %tmp4.1 = zext <2 x i32> %tmp4 to <2 x i64>
  %tmp4.1.1 = zext <2 x i32> %tmp4 to <2 x i64>
  %tmp5 = add <2 x i64> %tmp3, %tmp4.1
  ret <2 x i64> %tmp5
}

define <8 x i16> @sabal2_8h(<16 x i8>* %A, <16 x i8>* %B, <8 x i16>* %C) nounwind {
; CHECK-LABEL: sabal2_8h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x2]
; CHECK-NEXT:    ldr d1, [x0, #8]
; CHECK-NEXT:    ldr d2, [x1, #8]
; CHECK-NEXT:    sabal.8h v0, v1, v2
; CHECK-NEXT:    ret
  %load1 = load <16 x i8>, <16 x i8>* %A
  %load2 = load <16 x i8>, <16 x i8>* %B
  %tmp3 = load <8 x i16>, <8 x i16>* %C
  %tmp1 = shufflevector <16 x i8> %load1, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  %tmp2 = shufflevector <16 x i8> %load2, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  %tmp4 = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
  %tmp4.1 = zext <8 x i8> %tmp4 to <8 x i16>
  %tmp5 = add <8 x i16> %tmp3, %tmp4.1
  ret <8 x i16> %tmp5
}

define <4 x i32> @sabal2_4s(<8 x i16>* %A, <8 x i16>* %B, <4 x i32>* %C) nounwind {
; CHECK-LABEL: sabal2_4s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x2]
; CHECK-NEXT:    ldr d1, [x0, #8]
; CHECK-NEXT:    ldr d2, [x1, #8]
; CHECK-NEXT:    sabal.4s v0, v1, v2
; CHECK-NEXT:    ret
  %load1 = load <8 x i16>, <8 x i16>* %A
  %load2 = load <8 x i16>, <8 x i16>* %B
  %tmp3 = load <4 x i32>, <4 x i32>* %C
  %tmp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
  %tmp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
  %tmp4 = call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
  %tmp4.1 = zext <4 x i16> %tmp4 to <4 x i32>
  %tmp5 = add <4 x i32> %tmp3, %tmp4.1
  ret <4 x i32> %tmp5
}

define <2 x i64> @sabal2_2d(<4 x i32>* %A, <4 x i32>* %B, <2 x i64>* %C) nounwind {
; CHECK-LABEL: sabal2_2d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x2]
; CHECK-NEXT:    ldr d1, [x0, #8]
; CHECK-NEXT:    ldr d2, [x1, #8]
; CHECK-NEXT:    sabal.2d v0, v1, v2
; CHECK-NEXT:    ret
  %load1 = load <4 x i32>, <4 x i32>* %A
  %load2 = load <4 x i32>, <4 x i32>* %B
  %tmp3 = load <2 x i64>, <2 x i64>* %C
  %tmp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
  %tmp2 = shufflevector <4 x i32> %load2, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
  %tmp4 = call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
  %tmp4.1 = zext <2 x i32> %tmp4 to <2 x i64>
  %tmp5 = add <2 x i64> %tmp3, %tmp4.1
  ret <2 x i64> %tmp5
}

; FALLBACK-NOT: remark:{{.*}} uabal8h
define <8 x i16> @uabal8h(<8 x i8>* %A, <8 x i8>* %B,  <8 x i16>* %C) nounwind {
; DAG-LABEL: uabal8h:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr d1, [x1]
; DAG-NEXT:    ldr d2, [x0]
; DAG-NEXT:    ldr q0, [x2]
; DAG-NEXT:    uabal.8h v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: uabal8h:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr d1, [x0]
; GISEL-NEXT:    ldr d2, [x1]
; GISEL-NEXT:    ldr q0, [x2]
; GISEL-NEXT:    uabal.8h v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <8 x i8>, <8 x i8>* %A
  %tmp2 = load <8 x i8>, <8 x i8>* %B
  %tmp3 = load <8 x i16>, <8 x i16>* %C
  %tmp4 = call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
  %tmp4.1 = zext <8 x i8> %tmp4 to <8 x i16>
  %tmp5 = add <8 x i16> %tmp3, %tmp4.1
  ret <8 x i16> %tmp5
}

; FALLBACK-NOT: remark:{{.*}} uabal8s
define <4 x i32> @uabal4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
; DAG-LABEL: uabal4s:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr d1, [x1]
; DAG-NEXT:    ldr d2, [x0]
; DAG-NEXT:    ldr q0, [x2]
; DAG-NEXT:    uabal.4s v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: uabal4s:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr d1, [x0]
; GISEL-NEXT:    ldr d2, [x1]
; GISEL-NEXT:    ldr q0, [x2]
; GISEL-NEXT:    uabal.4s v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <4 x i16>, <4 x i16>* %A
  %tmp2 = load <4 x i16>, <4 x i16>* %B
  %tmp3 = load <4 x i32>, <4 x i32>* %C
  %tmp4 = call <4 x i16> @llvm.aarch64.neon.uabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
  %tmp4.1 = zext <4 x i16> %tmp4 to <4 x i32>
  %tmp5 = add <4 x i32> %tmp3, %tmp4.1
  ret <4 x i32> %tmp5
}

; FALLBACK-NOT: remark:{{.*}} uabal2d
define <2 x i64> @uabal2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
; DAG-LABEL: uabal2d:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr d1, [x1]
; DAG-NEXT:    ldr d2, [x0]
; DAG-NEXT:    ldr q0, [x2]
; DAG-NEXT:    uabal.2d v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: uabal2d:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr d1, [x0]
; GISEL-NEXT:    ldr d2, [x1]
; GISEL-NEXT:    ldr q0, [x2]
; GISEL-NEXT:    uabal.2d v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <2 x i32>, <2 x i32>* %A
  %tmp2 = load <2 x i32>, <2 x i32>* %B
  %tmp3 = load <2 x i64>, <2 x i64>* %C
  %tmp4 = call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
  %tmp4.1 = zext <2 x i32> %tmp4 to <2 x i64>
  %tmp5 = add <2 x i64> %tmp3, %tmp4.1
  ret <2 x i64> %tmp5
}

define <8 x i16> @uabal2_8h(<16 x i8>* %A, <16 x i8>* %B, <8 x i16>* %C) nounwind {
; CHECK-LABEL: uabal2_8h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x2]
; CHECK-NEXT:    ldr d1, [x0, #8]
; CHECK-NEXT:    ldr d2, [x1, #8]
; CHECK-NEXT:    uabal.8h v0, v1, v2
; CHECK-NEXT:    ret
  %load1 = load <16 x i8>, <16 x i8>* %A
  %load2 = load <16 x i8>, <16 x i8>* %B
  %tmp3 = load <8 x i16>, <8 x i16>* %C
  %tmp1 = shufflevector <16 x i8> %load1, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  %tmp2 = shufflevector <16 x i8> %load2, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  %tmp4 = call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
  %tmp4.1 = zext <8 x i8> %tmp4 to <8 x i16>
  %tmp5 = add <8 x i16> %tmp3, %tmp4.1
  ret <8 x i16> %tmp5
}

define <4 x i32> @uabal2_4s(<8 x i16>* %A, <8 x i16>* %B, <4 x i32>* %C) nounwind {
; CHECK-LABEL: uabal2_4s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x2]
; CHECK-NEXT:    ldr d1, [x0, #8]
; CHECK-NEXT:    ldr d2, [x1, #8]
; CHECK-NEXT:    uabal.4s v0, v1, v2
; CHECK-NEXT:    ret
  %load1 = load <8 x i16>, <8 x i16>* %A
  %load2 = load <8 x i16>, <8 x i16>* %B
  %tmp3 = load <4 x i32>, <4 x i32>* %C
  %tmp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
  %tmp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
  %tmp4 = call <4 x i16> @llvm.aarch64.neon.uabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
  %tmp4.1 = zext <4 x i16> %tmp4 to <4 x i32>
  %tmp5 = add <4 x i32> %tmp3, %tmp4.1
  ret <4 x i32> %tmp5
}

define <2 x i64> @uabal2_2d(<4 x i32>* %A, <4 x i32>* %B, <2 x i64>* %C) nounwind {
; CHECK-LABEL: uabal2_2d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr q0, [x2]
; CHECK-NEXT:    ldr d1, [x0, #8]
; CHECK-NEXT:    ldr d2, [x1, #8]
; CHECK-NEXT:    uabal.2d v0, v1, v2
; CHECK-NEXT:    ret
  %load1 = load <4 x i32>, <4 x i32>* %A
  %load2 = load <4 x i32>, <4 x i32>* %B
  %tmp3 = load <2 x i64>, <2 x i64>* %C
  %tmp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
  %tmp2 = shufflevector <4 x i32> %load2, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
  %tmp4 = call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
  %tmp4.1 = zext <2 x i32> %tmp4 to <2 x i64>
  %tmp5 = add <2 x i64> %tmp3, %tmp4.1
  ret <2 x i64> %tmp5
}

define <8 x i8> @saba_8b(<8 x i8>* %A, <8 x i8>* %B, <8 x i8>* %C) nounwind {
; DAG-LABEL: saba_8b:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr d1, [x1]
; DAG-NEXT:    ldr d2, [x0]
; DAG-NEXT:    ldr d0, [x2]
; DAG-NEXT:    saba.8b v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: saba_8b:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr d1, [x0]
; GISEL-NEXT:    ldr d2, [x1]
; GISEL-NEXT:    ldr d0, [x2]
; GISEL-NEXT:    saba.8b v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <8 x i8>, <8 x i8>* %A
  %tmp2 = load <8 x i8>, <8 x i8>* %B
  %tmp3 = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
  %tmp4 = load <8 x i8>, <8 x i8>* %C
  %tmp5 = add <8 x i8> %tmp3, %tmp4
  ret <8 x i8> %tmp5
}

define <16 x i8> @saba_16b(<16 x i8>* %A, <16 x i8>* %B, <16 x i8>* %C) nounwind {
; DAG-LABEL: saba_16b:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr q1, [x1]
; DAG-NEXT:    ldr q2, [x0]
; DAG-NEXT:    ldr q0, [x2]
; DAG-NEXT:    saba.16b v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: saba_16b:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr q1, [x0]
; GISEL-NEXT:    ldr q2, [x1]
; GISEL-NEXT:    ldr q0, [x2]
; GISEL-NEXT:    saba.16b v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <16 x i8>, <16 x i8>* %A
  %tmp2 = load <16 x i8>, <16 x i8>* %B
  %tmp3 = call <16 x i8> @llvm.aarch64.neon.sabd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
  %tmp4 = load <16 x i8>, <16 x i8>* %C
  %tmp5 = add <16 x i8> %tmp3, %tmp4
  ret <16 x i8> %tmp5
}

define <4 x i16> @saba_4h(<4 x i16>* %A, <4 x i16>* %B, <4 x i16>* %C) nounwind {
; DAG-LABEL: saba_4h:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr d1, [x1]
; DAG-NEXT:    ldr d2, [x0]
; DAG-NEXT:    ldr d0, [x2]
; DAG-NEXT:    saba.4h v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: saba_4h:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr d1, [x0]
; GISEL-NEXT:    ldr d2, [x1]
; GISEL-NEXT:    ldr d0, [x2]
; GISEL-NEXT:    saba.4h v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <4 x i16>, <4 x i16>* %A
  %tmp2 = load <4 x i16>, <4 x i16>* %B
  %tmp3 = call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
  %tmp4 = load <4 x i16>, <4 x i16>* %C
  %tmp5 = add <4 x i16> %tmp3, %tmp4
  ret <4 x i16> %tmp5
}

define <8 x i16> @saba_8h(<8 x i16>* %A, <8 x i16>* %B, <8 x i16>* %C) nounwind {
; DAG-LABEL: saba_8h:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr q1, [x1]
; DAG-NEXT:    ldr q2, [x0]
; DAG-NEXT:    ldr q0, [x2]
; DAG-NEXT:    saba.8h v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: saba_8h:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr q1, [x0]
; GISEL-NEXT:    ldr q2, [x1]
; GISEL-NEXT:    ldr q0, [x2]
; GISEL-NEXT:    saba.8h v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <8 x i16>, <8 x i16>* %A
  %tmp2 = load <8 x i16>, <8 x i16>* %B
  %tmp3 = call <8 x i16> @llvm.aarch64.neon.sabd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
  %tmp4 = load <8 x i16>, <8 x i16>* %C
  %tmp5 = add <8 x i16> %tmp3, %tmp4
  ret <8 x i16> %tmp5
}

define <2 x i32> @saba_2s(<2 x i32>* %A, <2 x i32>* %B, <2 x i32>* %C) nounwind {
; DAG-LABEL: saba_2s:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr d1, [x1]
; DAG-NEXT:    ldr d2, [x0]
; DAG-NEXT:    ldr d0, [x2]
; DAG-NEXT:    saba.2s v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: saba_2s:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr d1, [x0]
; GISEL-NEXT:    ldr d2, [x1]
; GISEL-NEXT:    ldr d0, [x2]
; GISEL-NEXT:    saba.2s v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <2 x i32>, <2 x i32>* %A
  %tmp2 = load <2 x i32>, <2 x i32>* %B
  %tmp3 = call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
  %tmp4 = load <2 x i32>, <2 x i32>* %C
  %tmp5 = add <2 x i32> %tmp3, %tmp4
  ret <2 x i32> %tmp5
}

define <4 x i32> @saba_4s(<4 x i32>* %A, <4 x i32>* %B, <4 x i32>* %C) nounwind {
; DAG-LABEL: saba_4s:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr q1, [x1]
; DAG-NEXT:    ldr q2, [x0]
; DAG-NEXT:    ldr q0, [x2]
; DAG-NEXT:    saba.4s v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: saba_4s:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr q1, [x0]
; GISEL-NEXT:    ldr q2, [x1]
; GISEL-NEXT:    ldr q0, [x2]
; GISEL-NEXT:    saba.4s v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <4 x i32>, <4 x i32>* %A
  %tmp2 = load <4 x i32>, <4 x i32>* %B
  %tmp3 = call <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
  %tmp4 = load <4 x i32>, <4 x i32>* %C
  %tmp5 = add <4 x i32> %tmp3, %tmp4
  ret <4 x i32> %tmp5
}

define <8 x i8> @uaba_8b(<8 x i8>* %A, <8 x i8>* %B, <8 x i8>* %C) nounwind {
; DAG-LABEL: uaba_8b:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr d1, [x1]
; DAG-NEXT:    ldr d2, [x0]
; DAG-NEXT:    ldr d0, [x2]
; DAG-NEXT:    uaba.8b v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: uaba_8b:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr d1, [x0]
; GISEL-NEXT:    ldr d2, [x1]
; GISEL-NEXT:    ldr d0, [x2]
; GISEL-NEXT:    uaba.8b v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <8 x i8>, <8 x i8>* %A
  %tmp2 = load <8 x i8>, <8 x i8>* %B
  %tmp3 = call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
  %tmp4 = load <8 x i8>, <8 x i8>* %C
  %tmp5 = add <8 x i8> %tmp3, %tmp4
  ret <8 x i8> %tmp5
}

define <16 x i8> @uaba_16b(<16 x i8>* %A, <16 x i8>* %B, <16 x i8>* %C) nounwind {
; DAG-LABEL: uaba_16b:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr q1, [x1]
; DAG-NEXT:    ldr q2, [x0]
; DAG-NEXT:    ldr q0, [x2]
; DAG-NEXT:    uaba.16b v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: uaba_16b:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr q1, [x0]
; GISEL-NEXT:    ldr q2, [x1]
; GISEL-NEXT:    ldr q0, [x2]
; GISEL-NEXT:    uaba.16b v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <16 x i8>, <16 x i8>* %A
  %tmp2 = load <16 x i8>, <16 x i8>* %B
  %tmp3 = call <16 x i8> @llvm.aarch64.neon.uabd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)
  %tmp4 = load <16 x i8>, <16 x i8>* %C
  %tmp5 = add <16 x i8> %tmp3, %tmp4
  ret <16 x i8> %tmp5
}

define <4 x i16> @uaba_4h(<4 x i16>* %A, <4 x i16>* %B, <4 x i16>* %C) nounwind {
; DAG-LABEL: uaba_4h:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr d1, [x1]
; DAG-NEXT:    ldr d2, [x0]
; DAG-NEXT:    ldr d0, [x2]
; DAG-NEXT:    uaba.4h v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: uaba_4h:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr d1, [x0]
; GISEL-NEXT:    ldr d2, [x1]
; GISEL-NEXT:    ldr d0, [x2]
; GISEL-NEXT:    uaba.4h v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <4 x i16>, <4 x i16>* %A
  %tmp2 = load <4 x i16>, <4 x i16>* %B
  %tmp3 = call <4 x i16> @llvm.aarch64.neon.uabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
  %tmp4 = load <4 x i16>, <4 x i16>* %C
  %tmp5 = add <4 x i16> %tmp3, %tmp4
  ret <4 x i16> %tmp5
}

define <8 x i16> @uaba_8h(<8 x i16>* %A, <8 x i16>* %B, <8 x i16>* %C) nounwind {
; DAG-LABEL: uaba_8h:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr q1, [x1]
; DAG-NEXT:    ldr q2, [x0]
; DAG-NEXT:    ldr q0, [x2]
; DAG-NEXT:    uaba.8h v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: uaba_8h:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr q1, [x0]
; GISEL-NEXT:    ldr q2, [x1]
; GISEL-NEXT:    ldr q0, [x2]
; GISEL-NEXT:    uaba.8h v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <8 x i16>, <8 x i16>* %A
  %tmp2 = load <8 x i16>, <8 x i16>* %B
  %tmp3 = call <8 x i16> @llvm.aarch64.neon.uabd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)
  %tmp4 = load <8 x i16>, <8 x i16>* %C
  %tmp5 = add <8 x i16> %tmp3, %tmp4
  ret <8 x i16> %tmp5
}

define <2 x i32> @uaba_2s(<2 x i32>* %A, <2 x i32>* %B, <2 x i32>* %C) nounwind {
; DAG-LABEL: uaba_2s:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr d1, [x1]
; DAG-NEXT:    ldr d2, [x0]
; DAG-NEXT:    ldr d0, [x2]
; DAG-NEXT:    uaba.2s v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: uaba_2s:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr d1, [x0]
; GISEL-NEXT:    ldr d2, [x1]
; GISEL-NEXT:    ldr d0, [x2]
; GISEL-NEXT:    uaba.2s v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <2 x i32>, <2 x i32>* %A
  %tmp2 = load <2 x i32>, <2 x i32>* %B
  %tmp3 = call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
  %tmp4 = load <2 x i32>, <2 x i32>* %C
  %tmp5 = add <2 x i32> %tmp3, %tmp4
  ret <2 x i32> %tmp5
}

define <4 x i32> @uaba_4s(<4 x i32>* %A, <4 x i32>* %B, <4 x i32>* %C) nounwind {
; DAG-LABEL: uaba_4s:
; DAG:       // %bb.0:
; DAG-NEXT:    ldr q1, [x1]
; DAG-NEXT:    ldr q2, [x0]
; DAG-NEXT:    ldr q0, [x2]
; DAG-NEXT:    uaba.4s v0, v2, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: uaba_4s:
; GISEL:       // %bb.0:
; GISEL-NEXT:    ldr q1, [x0]
; GISEL-NEXT:    ldr q2, [x1]
; GISEL-NEXT:    ldr q0, [x2]
; GISEL-NEXT:    uaba.4s v0, v1, v2
; GISEL-NEXT:    ret
  %tmp1 = load <4 x i32>, <4 x i32>* %A
  %tmp2 = load <4 x i32>, <4 x i32>* %B
  %tmp3 = call <4 x i32> @llvm.aarch64.neon.uabd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)
  %tmp4 = load <4 x i32>, <4 x i32>* %C
  %tmp5 = add <4 x i32> %tmp3, %tmp4
  ret <4 x i32> %tmp5
}

; Scalar FABD
define float @fabds(float %a, float %b) nounwind {
; CHECK-LABEL: fabds:
; CHECK:       // %bb.0:
; CHECK-NEXT:    fabd s0, s0, s1
; CHECK-NEXT:    ret
  %vabd.i = tail call float @llvm.aarch64.sisd.fabd.f32(float %a, float %b) nounwind
  ret float %vabd.i
}

define double @fabdd(double %a, double %b) nounwind {
; CHECK-LABEL: fabdd:
; CHECK:       // %bb.0:
; CHECK-NEXT:    fabd d0, d0, d1
; CHECK-NEXT:    ret
  %vabd.i = tail call double @llvm.aarch64.sisd.fabd.f64(double %a, double %b) nounwind
  ret double %vabd.i
}

declare double @llvm.aarch64.sisd.fabd.f64(double, double) nounwind readnone
declare float @llvm.aarch64.sisd.fabd.f32(float, float) nounwind readnone

define float @fabds_from_fsub_fabs(float %a, float %b) nounwind {
; CHECK-LABEL: fabds_from_fsub_fabs:
; CHECK:       // %bb.0:
; CHECK-NEXT:    fabd s0, s0, s1
; CHECK-NEXT:    ret
  %sub = fsub float %a, %b
  %abs = tail call float @llvm.fabs.f32(float %sub)
  ret float %abs
}

define double @fabdd_from_fsub_fabs(double %a, double %b) nounwind {
; CHECK-LABEL: fabdd_from_fsub_fabs:
; CHECK:       // %bb.0:
; CHECK-NEXT:    fabd d0, d0, d1
; CHECK-NEXT:    ret
  %sub = fsub double %a, %b
  %abs = tail call double @llvm.fabs.f64(double %sub)
  ret double %abs
}

declare float @llvm.fabs.f32(float) nounwind readnone
declare double @llvm.fabs.f64(double) nounwind readnone

define <2 x i64> @uabdl_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
; CHECK-LABEL: uabdl_from_extract_dup:
; CHECK:       // %bb.0:
; CHECK-NEXT:    dup.2s v1, w0
; CHECK-NEXT:    uabdl.2d v0, v0, v1
; CHECK-NEXT:    ret
  %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1

  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>

  %res = tail call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind
  %res1 = zext <2 x i32> %res to <2 x i64>
  ret <2 x i64> %res1
}

define <2 x i64> @uabdl2_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
; CHECK-LABEL: uabdl2_from_extract_dup:
; CHECK:       // %bb.0:
; CHECK-NEXT:    dup.4s v1, w0
; CHECK-NEXT:    uabdl2.2d v0, v0, v1
; CHECK-NEXT:    ret
  %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1

  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>

  %res = tail call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind
  %res1 = zext <2 x i32> %res to <2 x i64>
  ret <2 x i64> %res1
}

define <2 x i64> @sabdl_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
; CHECK-LABEL: sabdl_from_extract_dup:
; CHECK:       // %bb.0:
; CHECK-NEXT:    dup.2s v1, w0
; CHECK-NEXT:    sabdl.2d v0, v0, v1
; CHECK-NEXT:    ret
  %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1

  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>

  %res = tail call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind
  %res1 = zext <2 x i32> %res to <2 x i64>
  ret <2 x i64> %res1
}

define <2 x i64> @sabdl2_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
; CHECK-LABEL: sabdl2_from_extract_dup:
; CHECK:       // %bb.0:
; CHECK-NEXT:    dup.4s v1, w0
; CHECK-NEXT:    sabdl2.2d v0, v0, v1
; CHECK-NEXT:    ret
  %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1

  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>

  %res = tail call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind
  %res1 = zext <2 x i32> %res to <2 x i64>
  ret <2 x i64> %res1
}

define <2 x i32> @abspattern1(<2 x i32> %a) nounwind {
; DAG-LABEL: abspattern1:
; DAG:       // %bb.0:
; DAG-NEXT:    abs.2s v0, v0
; DAG-NEXT:    ret
;
; GISEL-LABEL: abspattern1:
; GISEL:       // %bb.0:
; GISEL-NEXT:    movi.2d v1, #0000000000000000
; GISEL-NEXT:    neg.2s v2, v0
; GISEL-NEXT:    cmge.2s v1, v0, v1
; GISEL-NEXT:    shl.2s v1, v1, #31
; GISEL-NEXT:    sshr.2s v1, v1, #31
; GISEL-NEXT:    bif.8b v0, v2, v1
; GISEL-NEXT:    ret

  %tmp1neg = sub <2 x i32> zeroinitializer, %a
  %b = icmp sge <2 x i32> %a, zeroinitializer
  %abs = select <2 x i1> %b, <2 x i32> %a, <2 x i32> %tmp1neg
  ret <2 x i32> %abs
}

define <4 x i16> @abspattern2(<4 x i16> %a) nounwind {
; DAG-LABEL: abspattern2:
; DAG:       // %bb.0:
; DAG-NEXT:    abs.4h v0, v0
; DAG-NEXT:    ret
;
; GISEL-LABEL: abspattern2:
; GISEL:       // %bb.0:
; GISEL-NEXT:    movi.2d v1, #0000000000000000
; GISEL-NEXT:    neg.4h v2, v0
; GISEL-NEXT:    cmgt.4h v1, v0, v1
; GISEL-NEXT:    shl.4h v1, v1, #15
; GISEL-NEXT:    sshr.4h v1, v1, #15
; GISEL-NEXT:    bif.8b v0, v2, v1
; GISEL-NEXT:    ret
; For GlobalISel, this generates terrible code until we can pattern match this to abs.

  %tmp1neg = sub <4 x i16> zeroinitializer, %a
  %b = icmp sgt <4 x i16> %a, zeroinitializer
  %abs = select <4 x i1> %b, <4 x i16> %a, <4 x i16> %tmp1neg
  ret <4 x i16> %abs
}

define <8 x i8> @abspattern3(<8 x i8> %a) nounwind {
; DAG-LABEL: abspattern3:
; DAG:       // %bb.0:
; DAG-NEXT:    abs.8b v0, v0
; DAG-NEXT:    ret
;
; GISEL-LABEL: abspattern3:
; GISEL:       // %bb.0:
; GISEL-NEXT:    movi.2d v1, #0000000000000000
; GISEL-NEXT:    neg.8b v2, v0
; GISEL-NEXT:    cmgt.8b v1, v1, v0
; GISEL-NEXT:    shl.8b v1, v1, #7
; GISEL-NEXT:    sshr.8b v1, v1, #7
; GISEL-NEXT:    bit.8b v0, v2, v1
; GISEL-NEXT:    ret

  %tmp1neg = sub <8 x i8> zeroinitializer, %a
  %b = icmp slt <8 x i8> %a, zeroinitializer
  %abs = select <8 x i1> %b, <8 x i8> %tmp1neg, <8 x i8> %a
  ret <8 x i8> %abs
}

define <4 x i32> @abspattern4(<4 x i32> %a) nounwind {
; DAG-LABEL: abspattern4:
; DAG:       // %bb.0:
; DAG-NEXT:    abs.4s v0, v0
; DAG-NEXT:    ret
;
; GISEL-LABEL: abspattern4:
; GISEL:       // %bb.0:
; GISEL-NEXT:    movi.2d v1, #0000000000000000
; GISEL-NEXT:    neg.4s v2, v0
; GISEL-NEXT:    cmge.4s v1, v0, v1
; GISEL-NEXT:    shl.4s v1, v1, #31
; GISEL-NEXT:    sshr.4s v1, v1, #31
; GISEL-NEXT:    bif.16b v0, v2, v1
; GISEL-NEXT:    ret

  %tmp1neg = sub <4 x i32> zeroinitializer, %a
  %b = icmp sge <4 x i32> %a, zeroinitializer
  %abs = select <4 x i1> %b, <4 x i32> %a, <4 x i32> %tmp1neg
  ret <4 x i32> %abs
}

define <8 x i16> @abspattern5(<8 x i16> %a) nounwind {
; DAG-LABEL: abspattern5:
; DAG:       // %bb.0:
; DAG-NEXT:    abs.8h v0, v0
; DAG-NEXT:    ret
;
; GISEL-LABEL: abspattern5:
; GISEL:       // %bb.0:
; GISEL-NEXT:    movi.2d v1, #0000000000000000
; GISEL-NEXT:    neg.8h v2, v0
; GISEL-NEXT:    cmgt.8h v1, v0, v1
; GISEL-NEXT:    shl.8h v1, v1, #15
; GISEL-NEXT:    sshr.8h v1, v1, #15
; GISEL-NEXT:    bif.16b v0, v2, v1
; GISEL-NEXT:    ret

  %tmp1neg = sub <8 x i16> zeroinitializer, %a
  %b = icmp sgt <8 x i16> %a, zeroinitializer
  %abs = select <8 x i1> %b, <8 x i16> %a, <8 x i16> %tmp1neg
  ret <8 x i16> %abs
}

define <16 x i8> @abspattern6(<16 x i8> %a) nounwind {
; DAG-LABEL: abspattern6:
; DAG:       // %bb.0:
; DAG-NEXT:    abs.16b v0, v0
; DAG-NEXT:    ret
;
; GISEL-LABEL: abspattern6:
; GISEL:       // %bb.0:
; GISEL-NEXT:    movi.2d v1, #0000000000000000
; GISEL-NEXT:    neg.16b v2, v0
; GISEL-NEXT:    cmgt.16b v1, v1, v0
; GISEL-NEXT:    shl.16b v1, v1, #7
; GISEL-NEXT:    sshr.16b v1, v1, #7
; GISEL-NEXT:    bit.16b v0, v2, v1
; GISEL-NEXT:    ret

  %tmp1neg = sub <16 x i8> zeroinitializer, %a
  %b = icmp slt <16 x i8> %a, zeroinitializer
  %abs = select <16 x i1> %b, <16 x i8> %tmp1neg, <16 x i8> %a
  ret <16 x i8> %abs
}

define <2 x i64> @abspattern7(<2 x i64> %a) nounwind {
; DAG-LABEL: abspattern7:
; DAG:       // %bb.0:
; DAG-NEXT:    abs.2d v0, v0
; DAG-NEXT:    ret
;
; GISEL-LABEL: abspattern7:
; GISEL:       // %bb.0:
; GISEL-NEXT:    movi.2d v1, #0000000000000000
; GISEL-NEXT:    neg.2d v2, v0
; GISEL-NEXT:    cmge.2d v1, v1, v0
; GISEL-NEXT:    shl.2d v1, v1, #63
; GISEL-NEXT:    sshr.2d v1, v1, #63
; GISEL-NEXT:    bit.16b v0, v2, v1
; GISEL-NEXT:    ret

  %tmp1neg = sub <2 x i64> zeroinitializer, %a
  %b = icmp sle <2 x i64> %a, zeroinitializer
  %abs = select <2 x i1> %b, <2 x i64> %tmp1neg, <2 x i64> %a
  ret <2 x i64> %abs
}

define <2 x i64> @uabd_i32(<2 x i32> %a, <2 x i32> %b) {
; DAG-LABEL: uabd_i32:
; DAG:       // %bb.0:
; DAG-NEXT:    sabdl.2d v0, v0, v1
; DAG-NEXT:    ret
;
; GISEL-LABEL: uabd_i32:
; GISEL:       // %bb.0:
; GISEL-NEXT:    movi.2d v2, #0000000000000000
; GISEL-NEXT:    ssubl.2d v0, v0, v1
; GISEL-NEXT:    cmgt.2d v1, v2, v0
; GISEL-NEXT:    neg.2d v2, v0
; GISEL-NEXT:    shl.2d v1, v1, #63
; GISEL-NEXT:    sshr.2d v1, v1, #63
; GISEL-NEXT:    bit.16b v0, v2, v1
; GISEL-NEXT:    ret
  %aext = sext <2 x i32> %a to <2 x i64>
  %bext = sext <2 x i32> %b to <2 x i64>
  %abdiff = sub nsw <2 x i64> %aext, %bext
  %abcmp = icmp slt <2 x i64> %abdiff, zeroinitializer
  %ababs = sub nsw <2 x i64> zeroinitializer, %abdiff
  %absel = select <2 x i1> %abcmp, <2 x i64> %ababs, <2 x i64> %abdiff
  ret <2 x i64> %absel
}


define <2 x i128> @uabd_i64(<2 x i64> %a, <2 x i64> %b) {
; CHECK-LABEL: uabd_i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    fmov x9, d0
; CHECK-NEXT:    fmov x11, d1
; CHECK-NEXT:    mov.d x8, v0[1]
; CHECK-NEXT:    mov.d x10, v1[1]
; CHECK-NEXT:    asr x12, x9, #63
; CHECK-NEXT:    asr x13, x11, #63
; CHECK-NEXT:    subs x9, x9, x11
; CHECK-NEXT:    sbcs x11, x12, x13
; CHECK-NEXT:    asr x12, x8, #63
; CHECK-NEXT:    asr x13, x10, #63
; CHECK-NEXT:    subs x8, x8, x10
; CHECK-NEXT:    sbcs x10, x12, x13
; CHECK-NEXT:    negs x12, x8
; CHECK-NEXT:    ngcs x13, x10
; CHECK-NEXT:    cmp x10, #0
; CHECK-NEXT:    csel x2, x12, x8, lt
; CHECK-NEXT:    csel x3, x13, x10, lt
; CHECK-NEXT:    negs x8, x9
; CHECK-NEXT:    ngcs x10, x11
; CHECK-NEXT:    cmp x11, #0
; CHECK-NEXT:    csel x8, x8, x9, lt
; CHECK-NEXT:    csel x1, x10, x11, lt
; CHECK-NEXT:    fmov d0, x8
; CHECK-NEXT:    mov.d v0[1], x1
; CHECK-NEXT:    fmov x0, d0
; CHECK-NEXT:    ret
  %aext = sext <2 x i64> %a to <2 x i128>
  %bext = sext <2 x i64> %b to <2 x i128>
  %abdiff = sub nsw <2 x i128> %aext, %bext
  %abcmp = icmp slt <2 x i128> %abdiff, zeroinitializer
  %ababs = sub nsw <2 x i128> zeroinitializer, %abdiff
  %absel = select <2 x i1> %abcmp, <2 x i128> %ababs, <2 x i128> %abdiff
  ret <2 x i128> %absel
}