llvm/test/CodeGen/AArch64/overflow-vec.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple=aarch64-none-eabi -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI

; CHECK-GI:       warning: Instruction selection used fallback path for sadd_v32i8
; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for ssub_v32i8

define <2 x i8> @uadd_v2i8(<2 x i8> %a, <2 x i8> %b) {
; CHECK-SD-LABEL: uadd_v2i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    movi d2, #0x0000ff000000ff
; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    and v2.8b, v0.8b, v2.8b
; CHECK-SD-NEXT:    add v1.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    fmov d2, d1
; CHECK-SD-NEXT:    bic v2.2s, #1, lsl #8
; CHECK-SD-NEXT:    cmeq v2.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v2i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    add v2.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    mov w9, v1.s[1]
; CHECK-GI-NEXT:    mov w8, v2.s[1]
; CHECK-GI-NEXT:    mov v3.16b, v2.16b
; CHECK-GI-NEXT:    mov v1.b[1], w9
; CHECK-GI-NEXT:    mov v3.b[1], w8
; CHECK-GI-NEXT:    cmhi v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    umov w8, v1.b[0]
; CHECK-GI-NEXT:    umov w9, v1.b[1]
; CHECK-GI-NEXT:    fmov s1, w8
; CHECK-GI-NEXT:    mov v1.s[1], w9
; CHECK-GI-NEXT:    shl v1.2s, v1.2s, #31
; CHECK-GI-NEXT:    cmlt v1.2s, v1.2s, #0
; CHECK-GI-NEXT:    bit v0.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i8>, <2 x i1>} @llvm.uadd.with.overflow(<2 x i8> %a, <2 x i8> %b)
  %e0 = extractvalue {<2 x i8>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i8>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i8> %e0, <2 x i8> %a
  ret <2 x i8> %r
}

define <2 x i8> @sadd_v2i8(<2 x i8> %a, <2 x i8> %b) {
; CHECK-SD-LABEL: sadd_v2i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    shl v2.2s, v0.2s, #24
; CHECK-SD-NEXT:    shl v1.2s, v1.2s, #24
; CHECK-SD-NEXT:    sshr v2.2s, v2.2s, #24
; CHECK-SD-NEXT:    ssra v2.2s, v1.2s, #24
; CHECK-SD-NEXT:    shl v1.2s, v2.2s, #24
; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #24
; CHECK-SD-NEXT:    cmeq v1.2s, v1.2s, v2.2s
; CHECK-SD-NEXT:    bif v0.8b, v2.8b, v1.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v2i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    add v3.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    mov w9, v0.s[1]
; CHECK-GI-NEXT:    mov w10, v1.s[1]
; CHECK-GI-NEXT:    mov v5.16b, v0.16b
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    mov w8, v3.s[1]
; CHECK-GI-NEXT:    mov v4.16b, v3.16b
; CHECK-GI-NEXT:    mov v5.b[1], w9
; CHECK-GI-NEXT:    mov v1.b[1], w10
; CHECK-GI-NEXT:    mov v4.b[1], w8
; CHECK-GI-NEXT:    cmgt v1.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    cmgt v4.8b, v5.8b, v4.8b
; CHECK-GI-NEXT:    umov w8, v1.b[0]
; CHECK-GI-NEXT:    umov w10, v1.b[1]
; CHECK-GI-NEXT:    umov w9, v4.b[0]
; CHECK-GI-NEXT:    umov w11, v4.b[1]
; CHECK-GI-NEXT:    fmov s1, w8
; CHECK-GI-NEXT:    fmov s2, w9
; CHECK-GI-NEXT:    mov v1.s[1], w10
; CHECK-GI-NEXT:    mov v2.s[1], w11
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    shl v1.2s, v1.2s, #31
; CHECK-GI-NEXT:    cmlt v1.2s, v1.2s, #0
; CHECK-GI-NEXT:    bit v0.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i8>, <2 x i1>} @llvm.sadd.with.overflow(<2 x i8> %a, <2 x i8> %b)
  %e0 = extractvalue {<2 x i8>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i8>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i8> %e0, <2 x i8> %a
  ret <2 x i8> %r
}

define <2 x i8> @usub_v2i8(<2 x i8> %a, <2 x i8> %b) {
; CHECK-SD-LABEL: usub_v2i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    movi d2, #0x0000ff000000ff
; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    and v3.8b, v0.8b, v2.8b
; CHECK-SD-NEXT:    sub v1.2s, v3.2s, v1.2s
; CHECK-SD-NEXT:    and v2.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    cmeq v2.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v2i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    mov w8, v0.s[1]
; CHECK-GI-NEXT:    mov w9, v1.s[1]
; CHECK-GI-NEXT:    mov v2.16b, v0.16b
; CHECK-GI-NEXT:    sub v3.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    mov v2.b[1], w8
; CHECK-GI-NEXT:    mov v1.b[1], w9
; CHECK-GI-NEXT:    cmhi v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    umov w8, v1.b[0]
; CHECK-GI-NEXT:    umov w9, v1.b[1]
; CHECK-GI-NEXT:    fmov s1, w8
; CHECK-GI-NEXT:    mov v1.s[1], w9
; CHECK-GI-NEXT:    shl v1.2s, v1.2s, #31
; CHECK-GI-NEXT:    cmlt v1.2s, v1.2s, #0
; CHECK-GI-NEXT:    bit v0.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i8>, <2 x i1>} @llvm.usub.with.overflow(<2 x i8> %a, <2 x i8> %b)
  %e0 = extractvalue {<2 x i8>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i8>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i8> %e0, <2 x i8> %a
  ret <2 x i8> %r
}

define <2 x i8> @ssub_v2i8(<2 x i8> %a, <2 x i8> %b) {
; CHECK-SD-LABEL: ssub_v2i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    shl v1.2s, v1.2s, #24
; CHECK-SD-NEXT:    shl v2.2s, v0.2s, #24
; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #24
; CHECK-SD-NEXT:    sshr v2.2s, v2.2s, #24
; CHECK-SD-NEXT:    sub v1.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    shl v2.2s, v1.2s, #24
; CHECK-SD-NEXT:    sshr v2.2s, v2.2s, #24
; CHECK-SD-NEXT:    cmeq v2.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v2i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    sub v3.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    mov w8, v0.s[1]
; CHECK-GI-NEXT:    mov w9, v1.s[1]
; CHECK-GI-NEXT:    mov v4.16b, v0.16b
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    mov w10, v3.s[1]
; CHECK-GI-NEXT:    mov v5.16b, v3.16b
; CHECK-GI-NEXT:    mov v4.b[1], w8
; CHECK-GI-NEXT:    mov v1.b[1], w9
; CHECK-GI-NEXT:    mov v5.b[1], w10
; CHECK-GI-NEXT:    cmgt v1.8b, v1.8b, v4.8b
; CHECK-GI-NEXT:    cmgt v2.8b, v2.8b, v5.8b
; CHECK-GI-NEXT:    umov w8, v1.b[0]
; CHECK-GI-NEXT:    umov w10, v1.b[1]
; CHECK-GI-NEXT:    umov w9, v2.b[0]
; CHECK-GI-NEXT:    umov w11, v2.b[1]
; CHECK-GI-NEXT:    fmov s1, w8
; CHECK-GI-NEXT:    fmov s2, w9
; CHECK-GI-NEXT:    mov v1.s[1], w10
; CHECK-GI-NEXT:    mov v2.s[1], w11
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    shl v1.2s, v1.2s, #31
; CHECK-GI-NEXT:    cmlt v1.2s, v1.2s, #0
; CHECK-GI-NEXT:    bit v0.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i8>, <2 x i1>} @llvm.ssub.with.overflow(<2 x i8> %a, <2 x i8> %b)
  %e0 = extractvalue {<2 x i8>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i8>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i8> %e0, <2 x i8> %a
  ret <2 x i8> %r
}

define <2 x i8> @umul_v2i8(<2 x i8> %a, <2 x i8> %b) {
; CHECK-SD-LABEL: umul_v2i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    movi d2, #0x0000ff000000ff
; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    and v2.8b, v0.8b, v2.8b
; CHECK-SD-NEXT:    mul v1.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    ushr v2.2s, v1.2s, #8
; CHECK-SD-NEXT:    cmeq v2.2s, v2.2s, #0
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v2i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi d2, #0x0000ff000000ff
; CHECK-GI-NEXT:    and v3.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    and v2.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    mul v1.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    mul v2.2s, v3.2s, v2.2s
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    uzp1 v2.4h, v2.4h, v0.4h
; CHECK-GI-NEXT:    ushr v2.4h, v2.4h, #8
; CHECK-GI-NEXT:    uzp1 v2.8b, v2.8b, v0.8b
; CHECK-GI-NEXT:    cmeq v2.8b, v2.8b, v3.8b
; CHECK-GI-NEXT:    mvn v2.8b, v2.8b
; CHECK-GI-NEXT:    umov w8, v2.b[0]
; CHECK-GI-NEXT:    umov w9, v2.b[1]
; CHECK-GI-NEXT:    fmov s2, w8
; CHECK-GI-NEXT:    mov v2.s[1], w9
; CHECK-GI-NEXT:    shl v2.2s, v2.2s, #31
; CHECK-GI-NEXT:    cmlt v2.2s, v2.2s, #0
; CHECK-GI-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i8>, <2 x i1>} @llvm.umul.with.overflow(<2 x i8> %a, <2 x i8> %b)
  %e0 = extractvalue {<2 x i8>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i8>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i8> %e0, <2 x i8> %a
  ret <2 x i8> %r
}

define <2 x i8> @smul_v2i8(<2 x i8> %a, <2 x i8> %b) {
; CHECK-SD-LABEL: smul_v2i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    shl v1.2s, v1.2s, #24
; CHECK-SD-NEXT:    shl v2.2s, v0.2s, #24
; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #24
; CHECK-SD-NEXT:    sshr v2.2s, v2.2s, #24
; CHECK-SD-NEXT:    mul v1.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    shl v2.2s, v1.2s, #24
; CHECK-SD-NEXT:    sshr v2.2s, v2.2s, #24
; CHECK-SD-NEXT:    cmeq v2.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v2i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    shl v3.2s, v0.2s, #24
; CHECK-GI-NEXT:    shl v4.2s, v1.2s, #24
; CHECK-GI-NEXT:    mul v1.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    movi v2.8b, #15
; CHECK-GI-NEXT:    sshr v3.2s, v3.2s, #24
; CHECK-GI-NEXT:    sshr v4.2s, v4.2s, #24
; CHECK-GI-NEXT:    mov w8, v1.s[1]
; CHECK-GI-NEXT:    neg v2.8b, v2.8b
; CHECK-GI-NEXT:    mul v3.2s, v3.2s, v4.2s
; CHECK-GI-NEXT:    mov v4.16b, v1.16b
; CHECK-GI-NEXT:    mov v4.b[1], w8
; CHECK-GI-NEXT:    uzp1 v3.4h, v3.4h, v0.4h
; CHECK-GI-NEXT:    sshl v2.8b, v4.8b, v2.8b
; CHECK-GI-NEXT:    sshr v3.4h, v3.4h, #8
; CHECK-GI-NEXT:    uzp1 v3.8b, v3.8b, v0.8b
; CHECK-GI-NEXT:    cmeq v2.8b, v3.8b, v2.8b
; CHECK-GI-NEXT:    mvn v2.8b, v2.8b
; CHECK-GI-NEXT:    umov w8, v2.b[0]
; CHECK-GI-NEXT:    umov w9, v2.b[1]
; CHECK-GI-NEXT:    fmov s2, w8
; CHECK-GI-NEXT:    mov v2.s[1], w9
; CHECK-GI-NEXT:    shl v2.2s, v2.2s, #31
; CHECK-GI-NEXT:    cmlt v2.2s, v2.2s, #0
; CHECK-GI-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i8>, <2 x i1>} @llvm.smul.with.overflow(<2 x i8> %a, <2 x i8> %b)
  %e0 = extractvalue {<2 x i8>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i8>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i8> %e0, <2 x i8> %a
  ret <2 x i8> %r
}

define <3 x i8> @uadd_v3i8(<3 x i8> %a, <3 x i8> %b) {
; CHECK-SD-LABEL: uadd_v3i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    fmov s0, w0
; CHECK-SD-NEXT:    fmov s1, w3
; CHECK-SD-NEXT:    mov v0.h[1], w1
; CHECK-SD-NEXT:    mov v1.h[1], w4
; CHECK-SD-NEXT:    mov v0.h[2], w2
; CHECK-SD-NEXT:    mov v1.h[2], w5
; CHECK-SD-NEXT:    fmov d2, d0
; CHECK-SD-NEXT:    bic v1.4h, #255, lsl #8
; CHECK-SD-NEXT:    bic v2.4h, #255, lsl #8
; CHECK-SD-NEXT:    add v1.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    fmov d2, d1
; CHECK-SD-NEXT:    bic v2.4h, #1, lsl #8
; CHECK-SD-NEXT:    cmeq v2.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    umov w0, v0.h[0]
; CHECK-SD-NEXT:    umov w1, v0.h[1]
; CHECK-SD-NEXT:    umov w2, v0.h[2]
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v3i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    fmov s0, w3
; CHECK-GI-NEXT:    fmov s1, w0
; CHECK-GI-NEXT:    mov v2.16b, v0.16b
; CHECK-GI-NEXT:    mov v1.h[1], w1
; CHECK-GI-NEXT:    mov v0.b[1], w4
; CHECK-GI-NEXT:    mov v2.h[1], w4
; CHECK-GI-NEXT:    mov v1.h[2], w2
; CHECK-GI-NEXT:    mov v2.h[2], w5
; CHECK-GI-NEXT:    add v2.4h, v1.4h, v2.4h
; CHECK-GI-NEXT:    uzp1 v3.8b, v2.8b, v0.8b
; CHECK-GI-NEXT:    mov v0.b[2], w5
; CHECK-GI-NEXT:    cmhi v0.8b, v0.8b, v3.8b
; CHECK-GI-NEXT:    movi d3, #0xffffffffffffffff
; CHECK-GI-NEXT:    zip1 v0.8b, v0.8b, v0.8b
; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #15
; CHECK-GI-NEXT:    cmlt v0.4h, v0.4h, #0
; CHECK-GI-NEXT:    eor v3.8b, v0.8b, v3.8b
; CHECK-GI-NEXT:    and v0.8b, v2.8b, v0.8b
; CHECK-GI-NEXT:    and v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    umov w0, v0.h[0]
; CHECK-GI-NEXT:    umov w1, v0.h[1]
; CHECK-GI-NEXT:    umov w2, v0.h[2]
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i8>, <3 x i1>} @llvm.uadd.with.overflow(<3 x i8> %a, <3 x i8> %b)
  %e0 = extractvalue {<3 x i8>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i8>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i8> %e0, <3 x i8> %a
  ret <3 x i8> %r
}

define <3 x i8> @sadd_v3i8(<3 x i8> %a, <3 x i8> %b) {
; CHECK-SD-LABEL: sadd_v3i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    fmov s0, w0
; CHECK-SD-NEXT:    fmov s1, w3
; CHECK-SD-NEXT:    mov v0.h[1], w1
; CHECK-SD-NEXT:    mov v1.h[1], w4
; CHECK-SD-NEXT:    mov v0.h[2], w2
; CHECK-SD-NEXT:    mov v1.h[2], w5
; CHECK-SD-NEXT:    shl v2.4h, v0.4h, #8
; CHECK-SD-NEXT:    shl v1.4h, v1.4h, #8
; CHECK-SD-NEXT:    sshr v2.4h, v2.4h, #8
; CHECK-SD-NEXT:    ssra v2.4h, v1.4h, #8
; CHECK-SD-NEXT:    shl v1.4h, v2.4h, #8
; CHECK-SD-NEXT:    sshr v1.4h, v1.4h, #8
; CHECK-SD-NEXT:    cmeq v1.4h, v1.4h, v2.4h
; CHECK-SD-NEXT:    bif v0.8b, v2.8b, v1.8b
; CHECK-SD-NEXT:    umov w0, v0.h[0]
; CHECK-SD-NEXT:    umov w1, v0.h[1]
; CHECK-SD-NEXT:    umov w2, v0.h[2]
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v3i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    fmov s1, w0
; CHECK-GI-NEXT:    fmov s2, w3
; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v0.16b, v1.16b
; CHECK-GI-NEXT:    mov v3.16b, v2.16b
; CHECK-GI-NEXT:    mov v1.b[1], w1
; CHECK-GI-NEXT:    mov v2.b[1], w4
; CHECK-GI-NEXT:    mov v0.h[1], w1
; CHECK-GI-NEXT:    mov v3.h[1], w4
; CHECK-GI-NEXT:    mov v1.b[2], w2
; CHECK-GI-NEXT:    mov v2.b[2], w5
; CHECK-GI-NEXT:    mov v0.h[2], w2
; CHECK-GI-NEXT:    mov v3.h[2], w5
; CHECK-GI-NEXT:    cmgt v2.8b, v5.8b, v2.8b
; CHECK-GI-NEXT:    add v3.4h, v0.4h, v3.4h
; CHECK-GI-NEXT:    mov b5, v2.b[1]
; CHECK-GI-NEXT:    mov b7, v2.b[2]
; CHECK-GI-NEXT:    uzp1 v4.8b, v3.8b, v0.8b
; CHECK-GI-NEXT:    fmov w8, s5
; CHECK-GI-NEXT:    cmgt v1.8b, v1.8b, v4.8b
; CHECK-GI-NEXT:    mov v2.h[1], w8
; CHECK-GI-NEXT:    fmov w8, s7
; CHECK-GI-NEXT:    mov b4, v1.b[1]
; CHECK-GI-NEXT:    mov b6, v1.b[2]
; CHECK-GI-NEXT:    mov v2.h[2], w8
; CHECK-GI-NEXT:    fmov w9, s4
; CHECK-GI-NEXT:    mov v1.h[1], w9
; CHECK-GI-NEXT:    fmov w9, s6
; CHECK-GI-NEXT:    mov v1.h[2], w9
; CHECK-GI-NEXT:    eor v1.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    movi d2, #0xffffffffffffffff
; CHECK-GI-NEXT:    shl v1.4h, v1.4h, #15
; CHECK-GI-NEXT:    cmlt v1.4h, v1.4h, #0
; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    umov w0, v0.h[0]
; CHECK-GI-NEXT:    umov w1, v0.h[1]
; CHECK-GI-NEXT:    umov w2, v0.h[2]
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i8>, <3 x i1>} @llvm.sadd.with.overflow(<3 x i8> %a, <3 x i8> %b)
  %e0 = extractvalue {<3 x i8>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i8>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i8> %e0, <3 x i8> %a
  ret <3 x i8> %r
}

define <3 x i8> @usub_v3i8(<3 x i8> %a, <3 x i8> %b) {
; CHECK-SD-LABEL: usub_v3i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    fmov s0, w0
; CHECK-SD-NEXT:    fmov s1, w3
; CHECK-SD-NEXT:    mov v0.h[1], w1
; CHECK-SD-NEXT:    mov v1.h[1], w4
; CHECK-SD-NEXT:    mov v0.h[2], w2
; CHECK-SD-NEXT:    mov v1.h[2], w5
; CHECK-SD-NEXT:    fmov d2, d0
; CHECK-SD-NEXT:    bic v1.4h, #255, lsl #8
; CHECK-SD-NEXT:    bic v2.4h, #255, lsl #8
; CHECK-SD-NEXT:    sub v1.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    fmov d2, d1
; CHECK-SD-NEXT:    bic v2.4h, #255, lsl #8
; CHECK-SD-NEXT:    cmeq v2.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    umov w0, v0.h[0]
; CHECK-SD-NEXT:    umov w1, v0.h[1]
; CHECK-SD-NEXT:    umov w2, v0.h[2]
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v3i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    fmov s0, w0
; CHECK-GI-NEXT:    fmov s1, w3
; CHECK-GI-NEXT:    mov v2.16b, v0.16b
; CHECK-GI-NEXT:    mov v3.16b, v1.16b
; CHECK-GI-NEXT:    mov v0.b[1], w1
; CHECK-GI-NEXT:    mov v1.b[1], w4
; CHECK-GI-NEXT:    mov v2.h[1], w1
; CHECK-GI-NEXT:    mov v3.h[1], w4
; CHECK-GI-NEXT:    mov v0.b[2], w2
; CHECK-GI-NEXT:    mov v1.b[2], w5
; CHECK-GI-NEXT:    mov v2.h[2], w2
; CHECK-GI-NEXT:    mov v3.h[2], w5
; CHECK-GI-NEXT:    cmhi v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    movi d1, #0xffffffffffffffff
; CHECK-GI-NEXT:    sub v3.4h, v2.4h, v3.4h
; CHECK-GI-NEXT:    zip1 v0.8b, v0.8b, v0.8b
; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #15
; CHECK-GI-NEXT:    cmlt v0.4h, v0.4h, #0
; CHECK-GI-NEXT:    eor v1.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    and v0.8b, v3.8b, v0.8b
; CHECK-GI-NEXT:    and v1.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    umov w0, v0.h[0]
; CHECK-GI-NEXT:    umov w1, v0.h[1]
; CHECK-GI-NEXT:    umov w2, v0.h[2]
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i8>, <3 x i1>} @llvm.usub.with.overflow(<3 x i8> %a, <3 x i8> %b)
  %e0 = extractvalue {<3 x i8>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i8>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i8> %e0, <3 x i8> %a
  ret <3 x i8> %r
}

define <3 x i8> @ssub_v3i8(<3 x i8> %a, <3 x i8> %b) {
; CHECK-SD-LABEL: ssub_v3i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    fmov s0, w3
; CHECK-SD-NEXT:    fmov s1, w0
; CHECK-SD-NEXT:    mov v0.h[1], w4
; CHECK-SD-NEXT:    mov v1.h[1], w1
; CHECK-SD-NEXT:    mov v0.h[2], w5
; CHECK-SD-NEXT:    mov v1.h[2], w2
; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
; CHECK-SD-NEXT:    shl v2.4h, v1.4h, #8
; CHECK-SD-NEXT:    sshr v0.4h, v0.4h, #8
; CHECK-SD-NEXT:    sshr v2.4h, v2.4h, #8
; CHECK-SD-NEXT:    sub v0.4h, v2.4h, v0.4h
; CHECK-SD-NEXT:    shl v2.4h, v0.4h, #8
; CHECK-SD-NEXT:    sshr v2.4h, v2.4h, #8
; CHECK-SD-NEXT:    cmeq v2.4h, v2.4h, v0.4h
; CHECK-SD-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    umov w0, v0.h[0]
; CHECK-SD-NEXT:    umov w1, v0.h[1]
; CHECK-SD-NEXT:    umov w2, v0.h[2]
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v3i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    fmov s1, w0
; CHECK-GI-NEXT:    fmov s2, w3
; CHECK-GI-NEXT:    movi v5.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v0.16b, v1.16b
; CHECK-GI-NEXT:    mov v3.16b, v2.16b
; CHECK-GI-NEXT:    mov v1.b[1], w1
; CHECK-GI-NEXT:    mov v2.b[1], w4
; CHECK-GI-NEXT:    mov v0.h[1], w1
; CHECK-GI-NEXT:    mov v3.h[1], w4
; CHECK-GI-NEXT:    mov v1.b[2], w2
; CHECK-GI-NEXT:    mov v2.b[2], w5
; CHECK-GI-NEXT:    mov v0.h[2], w2
; CHECK-GI-NEXT:    mov v3.h[2], w5
; CHECK-GI-NEXT:    cmgt v1.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    sub v3.4h, v0.4h, v3.4h
; CHECK-GI-NEXT:    mov b6, v1.b[2]
; CHECK-GI-NEXT:    uzp1 v4.8b, v3.8b, v0.8b
; CHECK-GI-NEXT:    cmgt v2.8b, v5.8b, v4.8b
; CHECK-GI-NEXT:    mov b4, v1.b[1]
; CHECK-GI-NEXT:    mov b5, v2.b[1]
; CHECK-GI-NEXT:    mov b7, v2.b[2]
; CHECK-GI-NEXT:    fmov w8, s4
; CHECK-GI-NEXT:    fmov w9, s5
; CHECK-GI-NEXT:    mov v1.h[1], w8
; CHECK-GI-NEXT:    fmov w8, s6
; CHECK-GI-NEXT:    mov v2.h[1], w9
; CHECK-GI-NEXT:    fmov w9, s7
; CHECK-GI-NEXT:    mov v1.h[2], w8
; CHECK-GI-NEXT:    mov v2.h[2], w9
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    movi d2, #0xffffffffffffffff
; CHECK-GI-NEXT:    shl v1.4h, v1.4h, #15
; CHECK-GI-NEXT:    cmlt v1.4h, v1.4h, #0
; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    umov w0, v0.h[0]
; CHECK-GI-NEXT:    umov w1, v0.h[1]
; CHECK-GI-NEXT:    umov w2, v0.h[2]
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i8>, <3 x i1>} @llvm.ssub.with.overflow(<3 x i8> %a, <3 x i8> %b)
  %e0 = extractvalue {<3 x i8>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i8>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i8> %e0, <3 x i8> %a
  ret <3 x i8> %r
}

define <3 x i8> @umul_v3i8(<3 x i8> %a, <3 x i8> %b) {
; CHECK-SD-LABEL: umul_v3i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    fmov s0, w0
; CHECK-SD-NEXT:    fmov s1, w3
; CHECK-SD-NEXT:    mov v0.h[1], w1
; CHECK-SD-NEXT:    mov v1.h[1], w4
; CHECK-SD-NEXT:    mov v0.h[2], w2
; CHECK-SD-NEXT:    mov v1.h[2], w5
; CHECK-SD-NEXT:    fmov d2, d0
; CHECK-SD-NEXT:    bic v1.4h, #255, lsl #8
; CHECK-SD-NEXT:    bic v2.4h, #255, lsl #8
; CHECK-SD-NEXT:    mul v1.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    ushr v2.4h, v1.4h, #8
; CHECK-SD-NEXT:    cmeq v2.4h, v2.4h, #0
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    umov w0, v0.h[0]
; CHECK-SD-NEXT:    umov w1, v0.h[1]
; CHECK-SD-NEXT:    umov w2, v0.h[2]
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v3i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    and w8, w0, #0xff
; CHECK-GI-NEXT:    and w10, w3, #0xff
; CHECK-GI-NEXT:    and w9, w1, #0xff
; CHECK-GI-NEXT:    fmov s0, w8
; CHECK-GI-NEXT:    fmov s1, w10
; CHECK-GI-NEXT:    and w8, w4, #0xff
; CHECK-GI-NEXT:    fmov s2, w3
; CHECK-GI-NEXT:    movi d3, #0xffffffffffffffff
; CHECK-GI-NEXT:    mov v0.h[1], w9
; CHECK-GI-NEXT:    mov v1.h[1], w8
; CHECK-GI-NEXT:    and w8, w2, #0xff
; CHECK-GI-NEXT:    and w9, w5, #0xff
; CHECK-GI-NEXT:    mov v2.h[1], w4
; CHECK-GI-NEXT:    mov v0.h[2], w8
; CHECK-GI-NEXT:    mov v1.h[2], w9
; CHECK-GI-NEXT:    mov v2.h[2], w5
; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000
; CHECK-GI-NEXT:    ushr v0.4h, v0.4h, #8
; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
; CHECK-GI-NEXT:    cmeq v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    fmov s1, w0
; CHECK-GI-NEXT:    mvn v0.8b, v0.8b
; CHECK-GI-NEXT:    mov v1.h[1], w1
; CHECK-GI-NEXT:    zip1 v0.8b, v0.8b, v0.8b
; CHECK-GI-NEXT:    mov v1.h[2], w2
; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #15
; CHECK-GI-NEXT:    mul v2.4h, v1.4h, v2.4h
; CHECK-GI-NEXT:    cmlt v0.4h, v0.4h, #0
; CHECK-GI-NEXT:    eor v3.8b, v0.8b, v3.8b
; CHECK-GI-NEXT:    and v0.8b, v2.8b, v0.8b
; CHECK-GI-NEXT:    and v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    umov w0, v0.h[0]
; CHECK-GI-NEXT:    umov w1, v0.h[1]
; CHECK-GI-NEXT:    umov w2, v0.h[2]
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i8>, <3 x i1>} @llvm.umul.with.overflow(<3 x i8> %a, <3 x i8> %b)
  %e0 = extractvalue {<3 x i8>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i8>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i8> %e0, <3 x i8> %a
  ret <3 x i8> %r
}

define <3 x i8> @smul_v3i8(<3 x i8> %a, <3 x i8> %b) {
; CHECK-SD-LABEL: smul_v3i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    fmov s0, w3
; CHECK-SD-NEXT:    fmov s1, w0
; CHECK-SD-NEXT:    mov v0.h[1], w4
; CHECK-SD-NEXT:    mov v1.h[1], w1
; CHECK-SD-NEXT:    mov v0.h[2], w5
; CHECK-SD-NEXT:    mov v1.h[2], w2
; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
; CHECK-SD-NEXT:    shl v2.4h, v1.4h, #8
; CHECK-SD-NEXT:    sshr v0.4h, v0.4h, #8
; CHECK-SD-NEXT:    sshr v2.4h, v2.4h, #8
; CHECK-SD-NEXT:    mul v0.4h, v2.4h, v0.4h
; CHECK-SD-NEXT:    shl v2.4h, v0.4h, #8
; CHECK-SD-NEXT:    sshr v2.4h, v2.4h, #8
; CHECK-SD-NEXT:    cmeq v2.4h, v2.4h, v0.4h
; CHECK-SD-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    umov w0, v0.h[0]
; CHECK-SD-NEXT:    umov w1, v0.h[1]
; CHECK-SD-NEXT:    umov w2, v0.h[2]
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v3i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    lsl w8, w0, #8
; CHECK-GI-NEXT:    lsl w10, w3, #8
; CHECK-GI-NEXT:    lsl w9, w1, #8
; CHECK-GI-NEXT:    lsl w11, w4, #8
; CHECK-GI-NEXT:    fmov s0, w0
; CHECK-GI-NEXT:    fmov s1, w3
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    sbfx w10, w10, #8, #8
; CHECK-GI-NEXT:    sbfx w9, w9, #8, #8
; CHECK-GI-NEXT:    sbfx w11, w11, #8, #8
; CHECK-GI-NEXT:    lsl w12, w2, #8
; CHECK-GI-NEXT:    fmov s2, w8
; CHECK-GI-NEXT:    fmov s3, w10
; CHECK-GI-NEXT:    lsl w8, w5, #8
; CHECK-GI-NEXT:    mov v0.h[1], w1
; CHECK-GI-NEXT:    mov v1.h[1], w4
; CHECK-GI-NEXT:    sbfx w8, w8, #8, #8
; CHECK-GI-NEXT:    mov v2.h[1], w9
; CHECK-GI-NEXT:    mov v3.h[1], w11
; CHECK-GI-NEXT:    sbfx w9, w12, #8, #8
; CHECK-GI-NEXT:    mov v0.h[2], w2
; CHECK-GI-NEXT:    mov v1.h[2], w5
; CHECK-GI-NEXT:    mov v2.h[2], w9
; CHECK-GI-NEXT:    mov v3.h[2], w8
; CHECK-GI-NEXT:    mul v1.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    mul v2.4h, v2.4h, v3.4h
; CHECK-GI-NEXT:    movi v3.8b, #23
; CHECK-GI-NEXT:    uzp1 v4.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    sshr v2.4h, v2.4h, #8
; CHECK-GI-NEXT:    neg v3.8b, v3.8b
; CHECK-GI-NEXT:    uzp1 v2.8b, v2.8b, v0.8b
; CHECK-GI-NEXT:    sshl v3.8b, v4.8b, v3.8b
; CHECK-GI-NEXT:    cmeq v2.8b, v2.8b, v3.8b
; CHECK-GI-NEXT:    movi d3, #0xffffffffffffffff
; CHECK-GI-NEXT:    mvn v2.8b, v2.8b
; CHECK-GI-NEXT:    zip1 v2.8b, v2.8b, v2.8b
; CHECK-GI-NEXT:    shl v2.4h, v2.4h, #15
; CHECK-GI-NEXT:    cmlt v2.4h, v2.4h, #0
; CHECK-GI-NEXT:    eor v3.8b, v2.8b, v3.8b
; CHECK-GI-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v3.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    umov w0, v0.h[0]
; CHECK-GI-NEXT:    umov w1, v0.h[1]
; CHECK-GI-NEXT:    umov w2, v0.h[2]
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i8>, <3 x i1>} @llvm.smul.with.overflow(<3 x i8> %a, <3 x i8> %b)
  %e0 = extractvalue {<3 x i8>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i8>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i8> %e0, <3 x i8> %a
  ret <3 x i8> %r
}

define <4 x i8> @uadd_v4i8(<4 x i8> %a, <4 x i8> %b) {
; CHECK-SD-LABEL: uadd_v4i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    fmov d2, d0
; CHECK-SD-NEXT:    bic v1.4h, #255, lsl #8
; CHECK-SD-NEXT:    bic v2.4h, #255, lsl #8
; CHECK-SD-NEXT:    add v1.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    fmov d2, d1
; CHECK-SD-NEXT:    bic v2.4h, #1, lsl #8
; CHECK-SD-NEXT:    cmeq v2.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v4i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v3.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    movi d2, #0xff00ff00ff00ff
; CHECK-GI-NEXT:    uzp1 v4.8b, v3.8b, v0.8b
; CHECK-GI-NEXT:    cmhi v1.8b, v1.8b, v4.8b
; CHECK-GI-NEXT:    zip1 v1.8b, v1.8b, v1.8b
; CHECK-GI-NEXT:    shl v1.4h, v1.4h, #15
; CHECK-GI-NEXT:    cmlt v1.4h, v1.4h, #0
; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i8>, <4 x i1>} @llvm.uadd.with.overflow(<4 x i8> %a, <4 x i8> %b)
  %e0 = extractvalue {<4 x i8>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i8>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i8> %e0, <4 x i8> %a
  ret <4 x i8> %r
}

define <4 x i8> @sadd_v4i8(<4 x i8> %a, <4 x i8> %b) {
; CHECK-SD-LABEL: sadd_v4i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    shl v2.4h, v0.4h, #8
; CHECK-SD-NEXT:    shl v1.4h, v1.4h, #8
; CHECK-SD-NEXT:    sshr v2.4h, v2.4h, #8
; CHECK-SD-NEXT:    ssra v2.4h, v1.4h, #8
; CHECK-SD-NEXT:    shl v1.4h, v2.4h, #8
; CHECK-SD-NEXT:    sshr v1.4h, v1.4h, #8
; CHECK-SD-NEXT:    cmeq v1.4h, v1.4h, v2.4h
; CHECK-SD-NEXT:    bif v0.8b, v2.8b, v1.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v4i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v3.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    uzp1 v5.8b, v0.8b, v0.8b
; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    uzp1 v4.8b, v3.8b, v0.8b
; CHECK-GI-NEXT:    cmgt v1.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    cmgt v4.8b, v5.8b, v4.8b
; CHECK-GI-NEXT:    zip1 v1.8b, v1.8b, v1.8b
; CHECK-GI-NEXT:    zip1 v2.8b, v4.8b, v4.8b
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    movi d2, #0xff00ff00ff00ff
; CHECK-GI-NEXT:    shl v1.4h, v1.4h, #15
; CHECK-GI-NEXT:    cmlt v1.4h, v1.4h, #0
; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i8>, <4 x i1>} @llvm.sadd.with.overflow(<4 x i8> %a, <4 x i8> %b)
  %e0 = extractvalue {<4 x i8>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i8>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i8> %e0, <4 x i8> %a
  ret <4 x i8> %r
}

define <4 x i8> @usub_v4i8(<4 x i8> %a, <4 x i8> %b) {
; CHECK-SD-LABEL: usub_v4i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    fmov d2, d0
; CHECK-SD-NEXT:    bic v1.4h, #255, lsl #8
; CHECK-SD-NEXT:    bic v2.4h, #255, lsl #8
; CHECK-SD-NEXT:    sub v1.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    fmov d2, d1
; CHECK-SD-NEXT:    bic v2.4h, #255, lsl #8
; CHECK-SD-NEXT:    cmeq v2.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v4i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    uzp1 v3.8b, v0.8b, v0.8b
; CHECK-GI-NEXT:    uzp1 v4.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    movi d2, #0xff00ff00ff00ff
; CHECK-GI-NEXT:    sub v1.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    cmhi v3.8b, v4.8b, v3.8b
; CHECK-GI-NEXT:    zip1 v3.8b, v3.8b, v3.8b
; CHECK-GI-NEXT:    shl v3.4h, v3.4h, #15
; CHECK-GI-NEXT:    cmlt v3.4h, v3.4h, #0
; CHECK-GI-NEXT:    eor v2.8b, v3.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i8>, <4 x i1>} @llvm.usub.with.overflow(<4 x i8> %a, <4 x i8> %b)
  %e0 = extractvalue {<4 x i8>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i8>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i8> %e0, <4 x i8> %a
  ret <4 x i8> %r
}

define <4 x i8> @ssub_v4i8(<4 x i8> %a, <4 x i8> %b) {
; CHECK-SD-LABEL: ssub_v4i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    shl v1.4h, v1.4h, #8
; CHECK-SD-NEXT:    shl v2.4h, v0.4h, #8
; CHECK-SD-NEXT:    sshr v1.4h, v1.4h, #8
; CHECK-SD-NEXT:    sshr v2.4h, v2.4h, #8
; CHECK-SD-NEXT:    sub v1.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    shl v2.4h, v1.4h, #8
; CHECK-SD-NEXT:    sshr v2.4h, v2.4h, #8
; CHECK-SD-NEXT:    cmeq v2.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v4i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v3.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    uzp1 v4.8b, v0.8b, v0.8b
; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    uzp1 v5.8b, v3.8b, v0.8b
; CHECK-GI-NEXT:    cmgt v1.8b, v1.8b, v4.8b
; CHECK-GI-NEXT:    cmgt v2.8b, v2.8b, v5.8b
; CHECK-GI-NEXT:    zip1 v1.8b, v1.8b, v1.8b
; CHECK-GI-NEXT:    zip1 v2.8b, v2.8b, v2.8b
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    movi d2, #0xff00ff00ff00ff
; CHECK-GI-NEXT:    shl v1.4h, v1.4h, #15
; CHECK-GI-NEXT:    cmlt v1.4h, v1.4h, #0
; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i8>, <4 x i1>} @llvm.ssub.with.overflow(<4 x i8> %a, <4 x i8> %b)
  %e0 = extractvalue {<4 x i8>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i8>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i8> %e0, <4 x i8> %a
  ret <4 x i8> %r
}

define <4 x i8> @umul_v4i8(<4 x i8> %a, <4 x i8> %b) {
; CHECK-SD-LABEL: umul_v4i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    fmov d2, d0
; CHECK-SD-NEXT:    bic v1.4h, #255, lsl #8
; CHECK-SD-NEXT:    bic v2.4h, #255, lsl #8
; CHECK-SD-NEXT:    mul v1.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    ushr v2.4h, v1.4h, #8
; CHECK-SD-NEXT:    cmeq v2.4h, v2.4h, #0
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v4i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi d2, #0xff00ff00ff00ff
; CHECK-GI-NEXT:    and v3.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    and v4.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    mul v1.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    mul v3.4h, v3.4h, v4.4h
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    ushr v3.4h, v3.4h, #8
; CHECK-GI-NEXT:    uzp1 v3.8b, v3.8b, v0.8b
; CHECK-GI-NEXT:    cmeq v3.8b, v3.8b, v4.8b
; CHECK-GI-NEXT:    mvn v3.8b, v3.8b
; CHECK-GI-NEXT:    zip1 v3.8b, v3.8b, v3.8b
; CHECK-GI-NEXT:    shl v3.4h, v3.4h, #15
; CHECK-GI-NEXT:    cmlt v3.4h, v3.4h, #0
; CHECK-GI-NEXT:    eor v2.8b, v3.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i8>, <4 x i1>} @llvm.umul.with.overflow(<4 x i8> %a, <4 x i8> %b)
  %e0 = extractvalue {<4 x i8>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i8>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i8> %e0, <4 x i8> %a
  ret <4 x i8> %r
}

define <4 x i8> @smul_v4i8(<4 x i8> %a, <4 x i8> %b) {
; CHECK-SD-LABEL: smul_v4i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    shl v1.4h, v1.4h, #8
; CHECK-SD-NEXT:    shl v2.4h, v0.4h, #8
; CHECK-SD-NEXT:    sshr v1.4h, v1.4h, #8
; CHECK-SD-NEXT:    sshr v2.4h, v2.4h, #8
; CHECK-SD-NEXT:    mul v1.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    shl v2.4h, v1.4h, #8
; CHECK-SD-NEXT:    sshr v2.4h, v2.4h, #8
; CHECK-SD-NEXT:    cmeq v2.4h, v2.4h, v1.4h
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v4i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    shl v3.4h, v0.4h, #8
; CHECK-GI-NEXT:    shl v4.4h, v1.4h, #8
; CHECK-GI-NEXT:    mul v1.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    movi v2.8b, #31
; CHECK-GI-NEXT:    sshr v3.4h, v3.4h, #8
; CHECK-GI-NEXT:    sshr v4.4h, v4.4h, #8
; CHECK-GI-NEXT:    neg v2.8b, v2.8b
; CHECK-GI-NEXT:    mul v3.4h, v3.4h, v4.4h
; CHECK-GI-NEXT:    uzp1 v4.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    sshr v3.4h, v3.4h, #8
; CHECK-GI-NEXT:    sshl v2.8b, v4.8b, v2.8b
; CHECK-GI-NEXT:    uzp1 v3.8b, v3.8b, v0.8b
; CHECK-GI-NEXT:    cmeq v2.8b, v3.8b, v2.8b
; CHECK-GI-NEXT:    movi d3, #0xff00ff00ff00ff
; CHECK-GI-NEXT:    mvn v2.8b, v2.8b
; CHECK-GI-NEXT:    zip1 v2.8b, v2.8b, v2.8b
; CHECK-GI-NEXT:    shl v2.4h, v2.4h, #15
; CHECK-GI-NEXT:    cmlt v2.4h, v2.4h, #0
; CHECK-GI-NEXT:    eor v3.8b, v2.8b, v3.8b
; CHECK-GI-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v3.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i8>, <4 x i1>} @llvm.smul.with.overflow(<4 x i8> %a, <4 x i8> %b)
  %e0 = extractvalue {<4 x i8>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i8>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i8> %e0, <4 x i8> %a
  ret <4 x i8> %r
}

define <8 x i8> @uadd_v8i8(<8 x i8> %a, <8 x i8> %b) {
; CHECK-SD-LABEL: uadd_v8i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    add v1.8b, v0.8b, v1.8b
; CHECK-SD-NEXT:    cmhi v2.8b, v0.8b, v1.8b
; CHECK-SD-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v8i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    cmhi v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    bit v0.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i8>, <8 x i1>} @llvm.uadd.with.overflow(<8 x i8> %a, <8 x i8> %b)
  %e0 = extractvalue {<8 x i8>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i8>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i8> %e0, <8 x i8> %a
  ret <8 x i8> %r
}

define <8 x i8> @sadd_v8i8(<8 x i8> %a, <8 x i8> %b) {
; CHECK-SD-LABEL: sadd_v8i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqadd v2.8b, v0.8b, v1.8b
; CHECK-SD-NEXT:    add v1.8b, v0.8b, v1.8b
; CHECK-SD-NEXT:    cmeq v2.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v8i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    cmlt v1.8b, v1.8b, #0
; CHECK-GI-NEXT:    cmgt v3.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    bit v0.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i8>, <8 x i1>} @llvm.sadd.with.overflow(<8 x i8> %a, <8 x i8> %b)
  %e0 = extractvalue {<8 x i8>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i8>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i8> %e0, <8 x i8> %a
  ret <8 x i8> %r
}

define <8 x i8> @usub_v8i8(<8 x i8> %a, <8 x i8> %b) {
; CHECK-SD-LABEL: usub_v8i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub v1.8b, v0.8b, v1.8b
; CHECK-SD-NEXT:    cmhi v2.8b, v1.8b, v0.8b
; CHECK-SD-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v8i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    cmhi v1.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    bit v0.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i8>, <8 x i1>} @llvm.usub.with.overflow(<8 x i8> %a, <8 x i8> %b)
  %e0 = extractvalue {<8 x i8>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i8>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i8> %e0, <8 x i8> %a
  ret <8 x i8> %r
}

define <8 x i8> @ssub_v8i8(<8 x i8> %a, <8 x i8> %b) {
; CHECK-SD-LABEL: ssub_v8i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqsub v2.8b, v0.8b, v1.8b
; CHECK-SD-NEXT:    sub v1.8b, v0.8b, v1.8b
; CHECK-SD-NEXT:    cmeq v2.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v8i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    cmgt v1.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    cmlt v3.8b, v2.8b, #0
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    bit v0.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i8>, <8 x i1>} @llvm.ssub.with.overflow(<8 x i8> %a, <8 x i8> %b)
  %e0 = extractvalue {<8 x i8>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i8>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i8> %e0, <8 x i8> %a
  ret <8 x i8> %r
}

define <8 x i8> @umul_v8i8(<8 x i8> %a, <8 x i8> %b) {
; CHECK-SD-LABEL: umul_v8i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    umull v1.8h, v0.8b, v1.8b
; CHECK-SD-NEXT:    shrn v2.8b, v1.8h, #8
; CHECK-SD-NEXT:    xtn v1.8b, v1.8h
; CHECK-SD-NEXT:    cmeq v2.8b, v2.8b, #0
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v8i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    umull v2.8h, v0.8b, v1.8b
; CHECK-GI-NEXT:    mul v1.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    shrn v2.8b, v2.8h, #8
; CHECK-GI-NEXT:    cmtst v2.8b, v2.8b, v2.8b
; CHECK-GI-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i8>, <8 x i1>} @llvm.umul.with.overflow(<8 x i8> %a, <8 x i8> %b)
  %e0 = extractvalue {<8 x i8>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i8>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i8> %e0, <8 x i8> %a
  ret <8 x i8> %r
}

define <8 x i8> @smul_v8i8(<8 x i8> %a, <8 x i8> %b) {
; CHECK-SD-LABEL: smul_v8i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    smull v1.8h, v0.8b, v1.8b
; CHECK-SD-NEXT:    xtn v2.8b, v1.8h
; CHECK-SD-NEXT:    shrn v1.8b, v1.8h, #8
; CHECK-SD-NEXT:    cmlt v3.8b, v2.8b, #0
; CHECK-SD-NEXT:    cmeq v1.8b, v1.8b, v3.8b
; CHECK-SD-NEXT:    bif v0.8b, v2.8b, v1.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v8i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v2.8b, #63
; CHECK-GI-NEXT:    smull v3.8h, v0.8b, v1.8b
; CHECK-GI-NEXT:    mul v1.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    neg v2.8b, v2.8b
; CHECK-GI-NEXT:    shrn v3.8b, v3.8h, #8
; CHECK-GI-NEXT:    sshl v2.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    cmeq v2.8b, v3.8b, v2.8b
; CHECK-GI-NEXT:    mvn v2.8b, v2.8b
; CHECK-GI-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i8>, <8 x i1>} @llvm.smul.with.overflow(<8 x i8> %a, <8 x i8> %b)
  %e0 = extractvalue {<8 x i8>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i8>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i8> %e0, <8 x i8> %a
  ret <8 x i8> %r
}

define <16 x i8> @uadd_v16i8(<16 x i8> %a, <16 x i8> %b) {
; CHECK-SD-LABEL: uadd_v16i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    add v1.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    cmhi v2.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v16i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    cmhi v1.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<16 x i8>, <16 x i1>} @llvm.uadd.with.overflow(<16 x i8> %a, <16 x i8> %b)
  %e0 = extractvalue {<16 x i8>, <16 x i1>} %o, 0
  %e1 = extractvalue {<16 x i8>, <16 x i1>} %o, 1
  %r = select <16 x i1> %e1, <16 x i8> %e0, <16 x i8> %a
  ret <16 x i8> %r
}

define <16 x i8> @sadd_v16i8(<16 x i8> %a, <16 x i8> %b) {
; CHECK-SD-LABEL: sadd_v16i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqadd v2.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    add v1.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    cmeq v2.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v16i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    cmlt v1.16b, v1.16b, #0
; CHECK-GI-NEXT:    cmgt v3.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    eor v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<16 x i8>, <16 x i1>} @llvm.sadd.with.overflow(<16 x i8> %a, <16 x i8> %b)
  %e0 = extractvalue {<16 x i8>, <16 x i1>} %o, 0
  %e1 = extractvalue {<16 x i8>, <16 x i1>} %o, 1
  %r = select <16 x i1> %e1, <16 x i8> %e0, <16 x i8> %a
  ret <16 x i8> %r
}

define <16 x i8> @usub_v16i8(<16 x i8> %a, <16 x i8> %b) {
; CHECK-SD-LABEL: usub_v16i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub v1.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    cmhi v2.16b, v1.16b, v0.16b
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v16i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    cmhi v1.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<16 x i8>, <16 x i1>} @llvm.usub.with.overflow(<16 x i8> %a, <16 x i8> %b)
  %e0 = extractvalue {<16 x i8>, <16 x i1>} %o, 0
  %e1 = extractvalue {<16 x i8>, <16 x i1>} %o, 1
  %r = select <16 x i1> %e1, <16 x i8> %e0, <16 x i8> %a
  ret <16 x i8> %r
}

define <16 x i8> @ssub_v16i8(<16 x i8> %a, <16 x i8> %b) {
; CHECK-SD-LABEL: ssub_v16i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqsub v2.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    sub v1.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    cmeq v2.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v16i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    cmgt v1.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    cmlt v3.16b, v2.16b, #0
; CHECK-GI-NEXT:    eor v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<16 x i8>, <16 x i1>} @llvm.ssub.with.overflow(<16 x i8> %a, <16 x i8> %b)
  %e0 = extractvalue {<16 x i8>, <16 x i1>} %o, 0
  %e1 = extractvalue {<16 x i8>, <16 x i1>} %o, 1
  %r = select <16 x i1> %e1, <16 x i8> %e0, <16 x i8> %a
  ret <16 x i8> %r
}

define <16 x i8> @umul_v16i8(<16 x i8> %a, <16 x i8> %b) {
; CHECK-SD-LABEL: umul_v16i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    umull2 v2.8h, v0.16b, v1.16b
; CHECK-SD-NEXT:    umull v3.8h, v0.8b, v1.8b
; CHECK-SD-NEXT:    mul v1.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    uzp2 v2.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    cmeq v2.16b, v2.16b, #0
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v16i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    umull2 v2.8h, v0.16b, v1.16b
; CHECK-GI-NEXT:    umull v3.8h, v0.8b, v1.8b
; CHECK-GI-NEXT:    mul v1.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    uzp2 v2.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    cmtst v2.16b, v2.16b, v2.16b
; CHECK-GI-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow(<16 x i8> %a, <16 x i8> %b)
  %e0 = extractvalue {<16 x i8>, <16 x i1>} %o, 0
  %e1 = extractvalue {<16 x i8>, <16 x i1>} %o, 1
  %r = select <16 x i1> %e1, <16 x i8> %e0, <16 x i8> %a
  ret <16 x i8> %r
}

define <16 x i8> @smul_v16i8(<16 x i8> %a, <16 x i8> %b) {
; CHECK-SD-LABEL: smul_v16i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    smull2 v2.8h, v0.16b, v1.16b
; CHECK-SD-NEXT:    smull v3.8h, v0.8b, v1.8b
; CHECK-SD-NEXT:    mul v1.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    uzp2 v2.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    cmlt v3.16b, v1.16b, #0
; CHECK-SD-NEXT:    cmeq v2.16b, v2.16b, v3.16b
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v16i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v2.16b, #127
; CHECK-GI-NEXT:    smull2 v3.8h, v0.16b, v1.16b
; CHECK-GI-NEXT:    smull v4.8h, v0.8b, v1.8b
; CHECK-GI-NEXT:    mul v1.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    neg v2.16b, v2.16b
; CHECK-GI-NEXT:    uzp2 v3.16b, v4.16b, v3.16b
; CHECK-GI-NEXT:    sshl v2.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    cmeq v2.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    mvn v2.16b, v2.16b
; CHECK-GI-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<16 x i8>, <16 x i1>} @llvm.smul.with.overflow(<16 x i8> %a, <16 x i8> %b)
  %e0 = extractvalue {<16 x i8>, <16 x i1>} %o, 0
  %e1 = extractvalue {<16 x i8>, <16 x i1>} %o, 1
  %r = select <16 x i1> %e1, <16 x i8> %e0, <16 x i8> %a
  ret <16 x i8> %r
}

define <32 x i8> @uadd_v32i8(<32 x i8> %a, <32 x i8> %b) {
; CHECK-SD-LABEL: uadd_v32i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    add v3.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    add v2.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    cmhi v4.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    cmhi v5.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    bit v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bit v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v32i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v4.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    add v5.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    cmhi v2.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    cmhi v3.16b, v3.16b, v5.16b
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v2.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v3.16b
; CHECK-GI-NEXT:    ret
  %o = call {<32 x i8>, <32 x i1>} @llvm.uadd.with.overflow(<32 x i8> %a, <32 x i8> %b)
  %e0 = extractvalue {<32 x i8>, <32 x i1>} %o, 0
  %e1 = extractvalue {<32 x i8>, <32 x i1>} %o, 1
  %r = select <32 x i1> %e1, <32 x i8> %e0, <32 x i8> %a
  ret <32 x i8> %r
}

define <32 x i8> @sadd_v32i8(<32 x i8> %a, <32 x i8> %b) {
; CHECK-LABEL: sadd_v32i8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sqadd v4.16b, v1.16b, v3.16b
; CHECK-NEXT:    sqadd v5.16b, v0.16b, v2.16b
; CHECK-NEXT:    add v3.16b, v1.16b, v3.16b
; CHECK-NEXT:    add v2.16b, v0.16b, v2.16b
; CHECK-NEXT:    cmeq v4.16b, v3.16b, v4.16b
; CHECK-NEXT:    cmeq v5.16b, v2.16b, v5.16b
; CHECK-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-NEXT:    ret
  %o = call {<32 x i8>, <32 x i1>} @llvm.sadd.with.overflow(<32 x i8> %a, <32 x i8> %b)
  %e0 = extractvalue {<32 x i8>, <32 x i1>} %o, 0
  %e1 = extractvalue {<32 x i8>, <32 x i1>} %o, 1
  %r = select <32 x i1> %e1, <32 x i8> %e0, <32 x i8> %a
  ret <32 x i8> %r
}

define <32 x i8> @usub_v32i8(<32 x i8> %a, <32 x i8> %b) {
; CHECK-SD-LABEL: usub_v32i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub v3.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    sub v2.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    cmhi v4.16b, v3.16b, v1.16b
; CHECK-SD-NEXT:    cmhi v5.16b, v2.16b, v0.16b
; CHECK-SD-NEXT:    bit v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bit v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v32i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v4.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    sub v5.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    cmhi v2.16b, v2.16b, v0.16b
; CHECK-GI-NEXT:    cmhi v3.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v2.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v3.16b
; CHECK-GI-NEXT:    ret
  %o = call {<32 x i8>, <32 x i1>} @llvm.usub.with.overflow(<32 x i8> %a, <32 x i8> %b)
  %e0 = extractvalue {<32 x i8>, <32 x i1>} %o, 0
  %e1 = extractvalue {<32 x i8>, <32 x i1>} %o, 1
  %r = select <32 x i1> %e1, <32 x i8> %e0, <32 x i8> %a
  ret <32 x i8> %r
}

define <32 x i8> @ssub_v32i8(<32 x i8> %a, <32 x i8> %b) {
; CHECK-LABEL: ssub_v32i8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sqsub v4.16b, v1.16b, v3.16b
; CHECK-NEXT:    sqsub v5.16b, v0.16b, v2.16b
; CHECK-NEXT:    sub v3.16b, v1.16b, v3.16b
; CHECK-NEXT:    sub v2.16b, v0.16b, v2.16b
; CHECK-NEXT:    cmeq v4.16b, v3.16b, v4.16b
; CHECK-NEXT:    cmeq v5.16b, v2.16b, v5.16b
; CHECK-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-NEXT:    ret
  %o = call {<32 x i8>, <32 x i1>} @llvm.ssub.with.overflow(<32 x i8> %a, <32 x i8> %b)
  %e0 = extractvalue {<32 x i8>, <32 x i1>} %o, 0
  %e1 = extractvalue {<32 x i8>, <32 x i1>} %o, 1
  %r = select <32 x i1> %e1, <32 x i8> %e0, <32 x i8> %a
  ret <32 x i8> %r
}

define <32 x i8> @umul_v32i8(<32 x i8> %a, <32 x i8> %b) {
; CHECK-SD-LABEL: umul_v32i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    umull2 v4.8h, v1.16b, v3.16b
; CHECK-SD-NEXT:    umull v5.8h, v1.8b, v3.8b
; CHECK-SD-NEXT:    umull2 v6.8h, v0.16b, v2.16b
; CHECK-SD-NEXT:    umull v7.8h, v0.8b, v2.8b
; CHECK-SD-NEXT:    mul v3.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    mul v2.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    uzp2 v4.16b, v5.16b, v4.16b
; CHECK-SD-NEXT:    uzp2 v5.16b, v7.16b, v6.16b
; CHECK-SD-NEXT:    cmeq v4.16b, v4.16b, #0
; CHECK-SD-NEXT:    cmeq v5.16b, v5.16b, #0
; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v32i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    umull v4.8h, v0.8b, v2.8b
; CHECK-GI-NEXT:    umull2 v5.8h, v0.16b, v2.16b
; CHECK-GI-NEXT:    umull v6.8h, v1.8b, v3.8b
; CHECK-GI-NEXT:    umull2 v7.8h, v1.16b, v3.16b
; CHECK-GI-NEXT:    mul v2.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    mul v3.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    uzp2 v4.16b, v4.16b, v5.16b
; CHECK-GI-NEXT:    uzp2 v5.16b, v6.16b, v7.16b
; CHECK-GI-NEXT:    cmtst v4.16b, v4.16b, v4.16b
; CHECK-GI-NEXT:    cmtst v5.16b, v5.16b, v5.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    bit v1.16b, v3.16b, v5.16b
; CHECK-GI-NEXT:    ret
  %o = call {<32 x i8>, <32 x i1>} @llvm.umul.with.overflow(<32 x i8> %a, <32 x i8> %b)
  %e0 = extractvalue {<32 x i8>, <32 x i1>} %o, 0
  %e1 = extractvalue {<32 x i8>, <32 x i1>} %o, 1
  %r = select <32 x i1> %e1, <32 x i8> %e0, <32 x i8> %a
  ret <32 x i8> %r
}

define <32 x i8> @smul_v32i8(<32 x i8> %a, <32 x i8> %b) {
; CHECK-SD-LABEL: smul_v32i8:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    smull2 v4.8h, v1.16b, v3.16b
; CHECK-SD-NEXT:    smull v5.8h, v1.8b, v3.8b
; CHECK-SD-NEXT:    smull2 v6.8h, v0.16b, v2.16b
; CHECK-SD-NEXT:    smull v7.8h, v0.8b, v2.8b
; CHECK-SD-NEXT:    mul v3.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    mul v2.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    uzp2 v4.16b, v5.16b, v4.16b
; CHECK-SD-NEXT:    uzp2 v5.16b, v7.16b, v6.16b
; CHECK-SD-NEXT:    cmlt v6.16b, v3.16b, #0
; CHECK-SD-NEXT:    cmlt v7.16b, v2.16b, #0
; CHECK-SD-NEXT:    cmeq v4.16b, v4.16b, v6.16b
; CHECK-SD-NEXT:    cmeq v5.16b, v5.16b, v7.16b
; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v32i8:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v4.2d, #0xffffffffffffffff
; CHECK-GI-NEXT:    smull v5.8h, v0.8b, v2.8b
; CHECK-GI-NEXT:    smull v6.8h, v1.8b, v3.8b
; CHECK-GI-NEXT:    smull2 v7.8h, v0.16b, v2.16b
; CHECK-GI-NEXT:    mul v2.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    smull2 v16.8h, v1.16b, v3.16b
; CHECK-GI-NEXT:    mul v3.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    neg v4.16b, v4.16b
; CHECK-GI-NEXT:    shrn v5.8b, v5.8h, #8
; CHECK-GI-NEXT:    shrn v6.8b, v6.8h, #8
; CHECK-GI-NEXT:    sshl v17.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    sshl v4.16b, v3.16b, v4.16b
; CHECK-GI-NEXT:    shrn2 v5.16b, v7.8h, #8
; CHECK-GI-NEXT:    shrn2 v6.16b, v16.8h, #8
; CHECK-GI-NEXT:    cmeq v5.16b, v5.16b, v17.16b
; CHECK-GI-NEXT:    cmeq v4.16b, v6.16b, v4.16b
; CHECK-GI-NEXT:    mvn v5.16b, v5.16b
; CHECK-GI-NEXT:    mvn v4.16b, v4.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v5.16b
; CHECK-GI-NEXT:    bit v1.16b, v3.16b, v4.16b
; CHECK-GI-NEXT:    ret
  %o = call {<32 x i8>, <32 x i1>} @llvm.smul.with.overflow(<32 x i8> %a, <32 x i8> %b)
  %e0 = extractvalue {<32 x i8>, <32 x i1>} %o, 0
  %e1 = extractvalue {<32 x i8>, <32 x i1>} %o, 1
  %r = select <32 x i1> %e1, <32 x i8> %e0, <32 x i8> %a
  ret <32 x i8> %r
}

define <2 x i16> @uadd_v2i16(<2 x i16> %a, <2 x i16> %b) {
; CHECK-SD-LABEL: uadd_v2i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    movi d2, #0x00ffff0000ffff
; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    and v2.8b, v0.8b, v2.8b
; CHECK-SD-NEXT:    add v1.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    fmov d2, d1
; CHECK-SD-NEXT:    bic v2.2s, #1, lsl #16
; CHECK-SD-NEXT:    cmeq v2.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v2i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi d2, #0x00ffff0000ffff
; CHECK-GI-NEXT:    add v3.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    and v4.8b, v3.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    cmhi v1.2s, v1.2s, v4.2s
; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i16>, <2 x i1>} @llvm.uadd.with.overflow(<2 x i16> %a, <2 x i16> %b)
  %e0 = extractvalue {<2 x i16>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i16>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i16> %e0, <2 x i16> %a
  ret <2 x i16> %r
}

define <2 x i16> @sadd_v2i16(<2 x i16> %a, <2 x i16> %b) {
; CHECK-SD-LABEL: sadd_v2i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    shl v2.2s, v0.2s, #16
; CHECK-SD-NEXT:    shl v1.2s, v1.2s, #16
; CHECK-SD-NEXT:    sshr v2.2s, v2.2s, #16
; CHECK-SD-NEXT:    ssra v2.2s, v1.2s, #16
; CHECK-SD-NEXT:    shl v1.2s, v2.2s, #16
; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #16
; CHECK-SD-NEXT:    cmeq v1.2s, v1.2s, v2.2s
; CHECK-SD-NEXT:    bif v0.8b, v2.8b, v1.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v2i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v3.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    shl v5.2s, v0.2s, #16
; CHECK-GI-NEXT:    shl v1.2s, v1.2s, #16
; CHECK-GI-NEXT:    shl v4.2s, v3.2s, #16
; CHECK-GI-NEXT:    sshr v5.2s, v5.2s, #16
; CHECK-GI-NEXT:    sshr v1.2s, v1.2s, #16
; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
; CHECK-GI-NEXT:    sshr v4.2s, v4.2s, #16
; CHECK-GI-NEXT:    cmgt v1.2s, v2.2s, v1.2s
; CHECK-GI-NEXT:    movi d2, #0x00ffff0000ffff
; CHECK-GI-NEXT:    cmgt v4.2s, v5.2s, v4.2s
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v4.8b
; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i16>, <2 x i1>} @llvm.sadd.with.overflow(<2 x i16> %a, <2 x i16> %b)
  %e0 = extractvalue {<2 x i16>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i16>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i16> %e0, <2 x i16> %a
  ret <2 x i16> %r
}

define <2 x i16> @usub_v2i16(<2 x i16> %a, <2 x i16> %b) {
; CHECK-SD-LABEL: usub_v2i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    movi d2, #0x00ffff0000ffff
; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    and v3.8b, v0.8b, v2.8b
; CHECK-SD-NEXT:    sub v1.2s, v3.2s, v1.2s
; CHECK-SD-NEXT:    and v2.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    cmeq v2.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v2i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi d2, #0x00ffff0000ffff
; CHECK-GI-NEXT:    sub v3.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    and v4.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    cmhi v1.2s, v1.2s, v4.2s
; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i16>, <2 x i1>} @llvm.usub.with.overflow(<2 x i16> %a, <2 x i16> %b)
  %e0 = extractvalue {<2 x i16>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i16>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i16> %e0, <2 x i16> %a
  ret <2 x i16> %r
}

define <2 x i16> @ssub_v2i16(<2 x i16> %a, <2 x i16> %b) {
; CHECK-SD-LABEL: ssub_v2i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    shl v1.2s, v1.2s, #16
; CHECK-SD-NEXT:    shl v2.2s, v0.2s, #16
; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #16
; CHECK-SD-NEXT:    sshr v2.2s, v2.2s, #16
; CHECK-SD-NEXT:    sub v1.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    shl v2.2s, v1.2s, #16
; CHECK-SD-NEXT:    sshr v2.2s, v2.2s, #16
; CHECK-SD-NEXT:    cmeq v2.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v2i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v3.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    shl v4.2s, v0.2s, #16
; CHECK-GI-NEXT:    shl v1.2s, v1.2s, #16
; CHECK-GI-NEXT:    shl v5.2s, v3.2s, #16
; CHECK-GI-NEXT:    sshr v4.2s, v4.2s, #16
; CHECK-GI-NEXT:    sshr v1.2s, v1.2s, #16
; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
; CHECK-GI-NEXT:    sshr v5.2s, v5.2s, #16
; CHECK-GI-NEXT:    cmgt v1.2s, v1.2s, v4.2s
; CHECK-GI-NEXT:    movi d4, #0x00ffff0000ffff
; CHECK-GI-NEXT:    cmgt v2.2s, v2.2s, v5.2s
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v4.8b
; CHECK-GI-NEXT:    and v1.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i16>, <2 x i1>} @llvm.ssub.with.overflow(<2 x i16> %a, <2 x i16> %b)
  %e0 = extractvalue {<2 x i16>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i16>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i16> %e0, <2 x i16> %a
  ret <2 x i16> %r
}

define <2 x i16> @umul_v2i16(<2 x i16> %a, <2 x i16> %b) {
; CHECK-SD-LABEL: umul_v2i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    movi d2, #0x00ffff0000ffff
; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    and v2.8b, v0.8b, v2.8b
; CHECK-SD-NEXT:    mul v1.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    ushr v2.2s, v1.2s, #16
; CHECK-SD-NEXT:    cmeq v2.2s, v2.2s, #0
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v2i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi d2, #0x00ffff0000ffff
; CHECK-GI-NEXT:    and v3.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    and v4.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    mul v1.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    mul v3.2s, v3.2s, v4.2s
; CHECK-GI-NEXT:    movi v4.2d, #0000000000000000
; CHECK-GI-NEXT:    ushr v3.2s, v3.2s, #16
; CHECK-GI-NEXT:    ushll v4.4s, v4.4h, #0
; CHECK-GI-NEXT:    cmeq v3.2s, v3.2s, v4.2s
; CHECK-GI-NEXT:    mvn v4.8b, v3.8b
; CHECK-GI-NEXT:    bic v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    eor v2.8b, v4.8b, v2.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i16>, <2 x i1>} @llvm.umul.with.overflow(<2 x i16> %a, <2 x i16> %b)
  %e0 = extractvalue {<2 x i16>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i16>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i16> %e0, <2 x i16> %a
  ret <2 x i16> %r
}

define <2 x i16> @smul_v2i16(<2 x i16> %a, <2 x i16> %b) {
; CHECK-SD-LABEL: smul_v2i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    shl v1.2s, v1.2s, #16
; CHECK-SD-NEXT:    shl v2.2s, v0.2s, #16
; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #16
; CHECK-SD-NEXT:    sshr v2.2s, v2.2s, #16
; CHECK-SD-NEXT:    mul v1.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    shl v2.2s, v1.2s, #16
; CHECK-SD-NEXT:    sshr v2.2s, v2.2s, #16
; CHECK-SD-NEXT:    cmeq v2.2s, v2.2s, v1.2s
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v2i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    shl v3.2s, v0.2s, #16
; CHECK-GI-NEXT:    shl v4.2s, v1.2s, #16
; CHECK-GI-NEXT:    mul v1.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    movi v2.4h, #31
; CHECK-GI-NEXT:    movi d5, #0x00ffff0000ffff
; CHECK-GI-NEXT:    sshr v3.2s, v3.2s, #16
; CHECK-GI-NEXT:    sshr v4.2s, v4.2s, #16
; CHECK-GI-NEXT:    neg v2.4h, v2.4h
; CHECK-GI-NEXT:    mul v3.2s, v3.2s, v4.2s
; CHECK-GI-NEXT:    uzp1 v4.4h, v1.4h, v0.4h
; CHECK-GI-NEXT:    sshl v2.4h, v4.4h, v2.4h
; CHECK-GI-NEXT:    sshr v3.2s, v3.2s, #16
; CHECK-GI-NEXT:    and v3.8b, v3.8b, v5.8b
; CHECK-GI-NEXT:    ushll v2.4s, v2.4h, #0
; CHECK-GI-NEXT:    cmeq v2.2s, v3.2s, v2.2s
; CHECK-GI-NEXT:    mvn v3.8b, v2.8b
; CHECK-GI-NEXT:    bic v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    eor v3.8b, v3.8b, v5.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v3.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i16>, <2 x i1>} @llvm.smul.with.overflow(<2 x i16> %a, <2 x i16> %b)
  %e0 = extractvalue {<2 x i16>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i16>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i16> %e0, <2 x i16> %a
  ret <2 x i16> %r
}

define <3 x i16> @uadd_v3i16(<3 x i16> %a, <3 x i16> %b) {
; CHECK-SD-LABEL: uadd_v3i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    add v1.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    cmhi v2.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v3i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v3.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    movi d2, #0xffffffffffffffff
; CHECK-GI-NEXT:    cmhi v1.4h, v1.4h, v3.4h
; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i16>, <3 x i1>} @llvm.uadd.with.overflow(<3 x i16> %a, <3 x i16> %b)
  %e0 = extractvalue {<3 x i16>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i16>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i16> %e0, <3 x i16> %a
  ret <3 x i16> %r
}

define <3 x i16> @sadd_v3i16(<3 x i16> %a, <3 x i16> %b) {
; CHECK-SD-LABEL: sadd_v3i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqadd v2.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    add v1.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    cmeq v2.4h, v1.4h, v2.4h
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v3i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    add v3.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    cmgt v4.4h, v0.4h, v3.4h
; CHECK-GI-NEXT:    cmgt v1.4h, v2.4h, v1.4h
; CHECK-GI-NEXT:    movi d2, #0xffffffffffffffff
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v4.8b
; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v1.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i16>, <3 x i1>} @llvm.sadd.with.overflow(<3 x i16> %a, <3 x i16> %b)
  %e0 = extractvalue {<3 x i16>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i16>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i16> %e0, <3 x i16> %a
  ret <3 x i16> %r
}

define <3 x i16> @usub_v3i16(<3 x i16> %a, <3 x i16> %b) {
; CHECK-SD-LABEL: usub_v3i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub v1.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    cmhi v2.4h, v1.4h, v0.4h
; CHECK-SD-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v3i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi d2, #0xffffffffffffffff
; CHECK-GI-NEXT:    cmhi v3.4h, v1.4h, v0.4h
; CHECK-GI-NEXT:    sub v1.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    and v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    eor v2.8b, v3.8b, v2.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i16>, <3 x i1>} @llvm.usub.with.overflow(<3 x i16> %a, <3 x i16> %b)
  %e0 = extractvalue {<3 x i16>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i16>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i16> %e0, <3 x i16> %a
  ret <3 x i16> %r
}

define <3 x i16> @ssub_v3i16(<3 x i16> %a, <3 x i16> %b) {
; CHECK-SD-LABEL: ssub_v3i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqsub v2.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    sub v1.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    cmeq v2.4h, v1.4h, v2.4h
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v3i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    sub v3.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    cmgt v1.4h, v1.4h, v0.4h
; CHECK-GI-NEXT:    movi d4, #0xffffffffffffffff
; CHECK-GI-NEXT:    cmgt v2.4h, v2.4h, v3.4h
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    eor v2.8b, v1.8b, v4.8b
; CHECK-GI-NEXT:    and v1.8b, v3.8b, v1.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i16>, <3 x i1>} @llvm.ssub.with.overflow(<3 x i16> %a, <3 x i16> %b)
  %e0 = extractvalue {<3 x i16>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i16>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i16> %e0, <3 x i16> %a
  ret <3 x i16> %r
}

define <3 x i16> @umul_v3i16(<3 x i16> %a, <3 x i16> %b) {
; CHECK-SD-LABEL: umul_v3i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    umull v1.4s, v0.4h, v1.4h
; CHECK-SD-NEXT:    shrn v2.4h, v1.4s, #16
; CHECK-SD-NEXT:    xtn v1.4h, v1.4s
; CHECK-SD-NEXT:    cmeq v2.4h, v2.4h, #0
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v3i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    umov w8, v0.h[0]
; CHECK-GI-NEXT:    umov w9, v1.h[0]
; CHECK-GI-NEXT:    umov w10, v0.h[1]
; CHECK-GI-NEXT:    umov w11, v1.h[1]
; CHECK-GI-NEXT:    fmov s2, w8
; CHECK-GI-NEXT:    fmov s3, w9
; CHECK-GI-NEXT:    umov w8, v0.h[2]
; CHECK-GI-NEXT:    umov w9, v1.h[2]
; CHECK-GI-NEXT:    mul v1.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    mov v2.s[1], w10
; CHECK-GI-NEXT:    mov v3.s[1], w11
; CHECK-GI-NEXT:    mov v2.s[2], w8
; CHECK-GI-NEXT:    mov v3.s[2], w9
; CHECK-GI-NEXT:    mul v2.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    movi v3.2d, #0000000000000000
; CHECK-GI-NEXT:    shrn v2.4h, v2.4s, #16
; CHECK-GI-NEXT:    cmeq v2.4h, v2.4h, v3.4h
; CHECK-GI-NEXT:    movi d3, #0xffffffffffffffff
; CHECK-GI-NEXT:    mvn v4.8b, v2.8b
; CHECK-GI-NEXT:    bic v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    eor v3.8b, v4.8b, v3.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v3.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i16>, <3 x i1>} @llvm.umul.with.overflow(<3 x i16> %a, <3 x i16> %b)
  %e0 = extractvalue {<3 x i16>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i16>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i16> %e0, <3 x i16> %a
  ret <3 x i16> %r
}

define <3 x i16> @smul_v3i16(<3 x i16> %a, <3 x i16> %b) {
; CHECK-SD-LABEL: smul_v3i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    smull v1.4s, v0.4h, v1.4h
; CHECK-SD-NEXT:    xtn v2.4h, v1.4s
; CHECK-SD-NEXT:    shrn v1.4h, v1.4s, #16
; CHECK-SD-NEXT:    cmlt v3.4h, v2.4h, #0
; CHECK-SD-NEXT:    cmeq v1.4h, v1.4h, v3.4h
; CHECK-SD-NEXT:    bif v0.8b, v2.8b, v1.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v3i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    smov w8, v0.h[0]
; CHECK-GI-NEXT:    smov w9, v1.h[0]
; CHECK-GI-NEXT:    smov w10, v0.h[1]
; CHECK-GI-NEXT:    smov w11, v1.h[1]
; CHECK-GI-NEXT:    movi v2.4h, #47
; CHECK-GI-NEXT:    fmov s3, w8
; CHECK-GI-NEXT:    fmov s4, w9
; CHECK-GI-NEXT:    smov w8, v0.h[2]
; CHECK-GI-NEXT:    smov w9, v1.h[2]
; CHECK-GI-NEXT:    mul v1.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    neg v2.4h, v2.4h
; CHECK-GI-NEXT:    mov v3.s[1], w10
; CHECK-GI-NEXT:    mov v4.s[1], w11
; CHECK-GI-NEXT:    sshl v2.4h, v1.4h, v2.4h
; CHECK-GI-NEXT:    mov v3.s[2], w8
; CHECK-GI-NEXT:    mov v4.s[2], w9
; CHECK-GI-NEXT:    mul v3.4s, v3.4s, v4.4s
; CHECK-GI-NEXT:    shrn v3.4h, v3.4s, #16
; CHECK-GI-NEXT:    cmeq v2.4h, v3.4h, v2.4h
; CHECK-GI-NEXT:    movi d3, #0xffffffffffffffff
; CHECK-GI-NEXT:    mvn v4.8b, v2.8b
; CHECK-GI-NEXT:    bic v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    eor v3.8b, v4.8b, v3.8b
; CHECK-GI-NEXT:    and v0.8b, v0.8b, v3.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i16>, <3 x i1>} @llvm.smul.with.overflow(<3 x i16> %a, <3 x i16> %b)
  %e0 = extractvalue {<3 x i16>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i16>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i16> %e0, <3 x i16> %a
  ret <3 x i16> %r
}

define <4 x i16> @uadd_v4i16(<4 x i16> %a, <4 x i16> %b) {
; CHECK-SD-LABEL: uadd_v4i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    add v1.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    cmhi v2.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v4i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    cmhi v1.4h, v1.4h, v2.4h
; CHECK-GI-NEXT:    bit v0.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i16>, <4 x i1>} @llvm.uadd.with.overflow(<4 x i16> %a, <4 x i16> %b)
  %e0 = extractvalue {<4 x i16>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i16>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i16> %e0, <4 x i16> %a
  ret <4 x i16> %r
}

define <4 x i16> @sadd_v4i16(<4 x i16> %a, <4 x i16> %b) {
; CHECK-SD-LABEL: sadd_v4i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqadd v2.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    add v1.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    cmeq v2.4h, v1.4h, v2.4h
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v4i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    cmlt v1.4h, v1.4h, #0
; CHECK-GI-NEXT:    cmgt v3.4h, v0.4h, v2.4h
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    bit v0.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i16>, <4 x i1>} @llvm.sadd.with.overflow(<4 x i16> %a, <4 x i16> %b)
  %e0 = extractvalue {<4 x i16>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i16>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i16> %e0, <4 x i16> %a
  ret <4 x i16> %r
}

define <4 x i16> @usub_v4i16(<4 x i16> %a, <4 x i16> %b) {
; CHECK-SD-LABEL: usub_v4i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub v1.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    cmhi v2.4h, v1.4h, v0.4h
; CHECK-SD-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v4i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    cmhi v1.4h, v1.4h, v0.4h
; CHECK-GI-NEXT:    bit v0.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i16>, <4 x i1>} @llvm.usub.with.overflow(<4 x i16> %a, <4 x i16> %b)
  %e0 = extractvalue {<4 x i16>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i16>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i16> %e0, <4 x i16> %a
  ret <4 x i16> %r
}

define <4 x i16> @ssub_v4i16(<4 x i16> %a, <4 x i16> %b) {
; CHECK-SD-LABEL: ssub_v4i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqsub v2.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    sub v1.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    cmeq v2.4h, v1.4h, v2.4h
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v4i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    cmgt v1.4h, v1.4h, v0.4h
; CHECK-GI-NEXT:    cmlt v3.4h, v2.4h, #0
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    bit v0.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i16>, <4 x i1>} @llvm.ssub.with.overflow(<4 x i16> %a, <4 x i16> %b)
  %e0 = extractvalue {<4 x i16>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i16>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i16> %e0, <4 x i16> %a
  ret <4 x i16> %r
}

define <4 x i16> @umul_v4i16(<4 x i16> %a, <4 x i16> %b) {
; CHECK-SD-LABEL: umul_v4i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    umull v1.4s, v0.4h, v1.4h
; CHECK-SD-NEXT:    shrn v2.4h, v1.4s, #16
; CHECK-SD-NEXT:    xtn v1.4h, v1.4s
; CHECK-SD-NEXT:    cmeq v2.4h, v2.4h, #0
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v4i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    umull v2.4s, v0.4h, v1.4h
; CHECK-GI-NEXT:    mul v1.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    shrn v2.4h, v2.4s, #16
; CHECK-GI-NEXT:    cmtst v2.4h, v2.4h, v2.4h
; CHECK-GI-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i16>, <4 x i1>} @llvm.umul.with.overflow(<4 x i16> %a, <4 x i16> %b)
  %e0 = extractvalue {<4 x i16>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i16>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i16> %e0, <4 x i16> %a
  ret <4 x i16> %r
}

define <4 x i16> @smul_v4i16(<4 x i16> %a, <4 x i16> %b) {
; CHECK-SD-LABEL: smul_v4i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    smull v1.4s, v0.4h, v1.4h
; CHECK-SD-NEXT:    xtn v2.4h, v1.4s
; CHECK-SD-NEXT:    shrn v1.4h, v1.4s, #16
; CHECK-SD-NEXT:    cmlt v3.4h, v2.4h, #0
; CHECK-SD-NEXT:    cmeq v1.4h, v1.4h, v3.4h
; CHECK-SD-NEXT:    bif v0.8b, v2.8b, v1.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v4i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v2.4h, #63
; CHECK-GI-NEXT:    smull v3.4s, v0.4h, v1.4h
; CHECK-GI-NEXT:    mul v1.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    neg v2.4h, v2.4h
; CHECK-GI-NEXT:    shrn v3.4h, v3.4s, #16
; CHECK-GI-NEXT:    sshl v2.4h, v1.4h, v2.4h
; CHECK-GI-NEXT:    cmeq v2.4h, v3.4h, v2.4h
; CHECK-GI-NEXT:    mvn v2.8b, v2.8b
; CHECK-GI-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i16>, <4 x i1>} @llvm.smul.with.overflow(<4 x i16> %a, <4 x i16> %b)
  %e0 = extractvalue {<4 x i16>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i16>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i16> %e0, <4 x i16> %a
  ret <4 x i16> %r
}

define <8 x i16> @uadd_v8i16(<8 x i16> %a, <8 x i16> %b) {
; CHECK-SD-LABEL: uadd_v8i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    add v1.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    cmhi v2.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v8i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.8h, v0.8h, v1.8h
; CHECK-GI-NEXT:    cmhi v1.8h, v1.8h, v2.8h
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i16>, <8 x i1>} @llvm.uadd.with.overflow(<8 x i16> %a, <8 x i16> %b)
  %e0 = extractvalue {<8 x i16>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i16>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i16> %e0, <8 x i16> %a
  ret <8 x i16> %r
}

define <8 x i16> @sadd_v8i16(<8 x i16> %a, <8 x i16> %b) {
; CHECK-SD-LABEL: sadd_v8i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqadd v2.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    add v1.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    cmeq v2.8h, v1.8h, v2.8h
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v8i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.8h, v0.8h, v1.8h
; CHECK-GI-NEXT:    cmlt v1.8h, v1.8h, #0
; CHECK-GI-NEXT:    cmgt v3.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    eor v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i16>, <8 x i1>} @llvm.sadd.with.overflow(<8 x i16> %a, <8 x i16> %b)
  %e0 = extractvalue {<8 x i16>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i16>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i16> %e0, <8 x i16> %a
  ret <8 x i16> %r
}

define <8 x i16> @usub_v8i16(<8 x i16> %a, <8 x i16> %b) {
; CHECK-SD-LABEL: usub_v8i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub v1.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    cmhi v2.8h, v1.8h, v0.8h
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v8i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.8h, v0.8h, v1.8h
; CHECK-GI-NEXT:    cmhi v1.8h, v1.8h, v0.8h
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i16>, <8 x i1>} @llvm.usub.with.overflow(<8 x i16> %a, <8 x i16> %b)
  %e0 = extractvalue {<8 x i16>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i16>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i16> %e0, <8 x i16> %a
  ret <8 x i16> %r
}

define <8 x i16> @ssub_v8i16(<8 x i16> %a, <8 x i16> %b) {
; CHECK-SD-LABEL: ssub_v8i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqsub v2.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    sub v1.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    cmeq v2.8h, v1.8h, v2.8h
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v8i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.8h, v0.8h, v1.8h
; CHECK-GI-NEXT:    cmgt v1.8h, v1.8h, v0.8h
; CHECK-GI-NEXT:    cmlt v3.8h, v2.8h, #0
; CHECK-GI-NEXT:    eor v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i16>, <8 x i1>} @llvm.ssub.with.overflow(<8 x i16> %a, <8 x i16> %b)
  %e0 = extractvalue {<8 x i16>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i16>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i16> %e0, <8 x i16> %a
  ret <8 x i16> %r
}

define <8 x i16> @umul_v8i16(<8 x i16> %a, <8 x i16> %b) {
; CHECK-SD-LABEL: umul_v8i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    umull2 v2.4s, v0.8h, v1.8h
; CHECK-SD-NEXT:    umull v3.4s, v0.4h, v1.4h
; CHECK-SD-NEXT:    mul v1.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    uzp2 v2.8h, v3.8h, v2.8h
; CHECK-SD-NEXT:    cmeq v2.8h, v2.8h, #0
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v8i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    umull2 v2.4s, v0.8h, v1.8h
; CHECK-GI-NEXT:    umull v3.4s, v0.4h, v1.4h
; CHECK-GI-NEXT:    mul v1.8h, v0.8h, v1.8h
; CHECK-GI-NEXT:    uzp2 v2.8h, v3.8h, v2.8h
; CHECK-GI-NEXT:    cmtst v2.8h, v2.8h, v2.8h
; CHECK-GI-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow(<8 x i16> %a, <8 x i16> %b)
  %e0 = extractvalue {<8 x i16>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i16>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i16> %e0, <8 x i16> %a
  ret <8 x i16> %r
}

define <8 x i16> @smul_v8i16(<8 x i16> %a, <8 x i16> %b) {
; CHECK-SD-LABEL: smul_v8i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    smull2 v2.4s, v0.8h, v1.8h
; CHECK-SD-NEXT:    smull v3.4s, v0.4h, v1.4h
; CHECK-SD-NEXT:    mul v1.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    uzp2 v2.8h, v3.8h, v2.8h
; CHECK-SD-NEXT:    cmlt v3.8h, v1.8h, #0
; CHECK-SD-NEXT:    cmeq v2.8h, v2.8h, v3.8h
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v8i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v2.8h, #127
; CHECK-GI-NEXT:    smull2 v3.4s, v0.8h, v1.8h
; CHECK-GI-NEXT:    smull v4.4s, v0.4h, v1.4h
; CHECK-GI-NEXT:    mul v1.8h, v0.8h, v1.8h
; CHECK-GI-NEXT:    neg v2.8h, v2.8h
; CHECK-GI-NEXT:    uzp2 v3.8h, v4.8h, v3.8h
; CHECK-GI-NEXT:    sshl v2.8h, v1.8h, v2.8h
; CHECK-GI-NEXT:    cmeq v2.8h, v3.8h, v2.8h
; CHECK-GI-NEXT:    mvn v2.16b, v2.16b
; CHECK-GI-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i16>, <8 x i1>} @llvm.smul.with.overflow(<8 x i16> %a, <8 x i16> %b)
  %e0 = extractvalue {<8 x i16>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i16>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i16> %e0, <8 x i16> %a
  ret <8 x i16> %r
}

define <16 x i16> @uadd_v16i16(<16 x i16> %a, <16 x i16> %b) {
; CHECK-SD-LABEL: uadd_v16i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    add v3.8h, v1.8h, v3.8h
; CHECK-SD-NEXT:    add v2.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    cmhi v4.8h, v1.8h, v3.8h
; CHECK-SD-NEXT:    cmhi v5.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    bit v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bit v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v16i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v4.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    add v5.8h, v1.8h, v3.8h
; CHECK-GI-NEXT:    cmhi v2.8h, v2.8h, v4.8h
; CHECK-GI-NEXT:    cmhi v3.8h, v3.8h, v5.8h
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v2.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v3.16b
; CHECK-GI-NEXT:    ret
  %o = call {<16 x i16>, <16 x i1>} @llvm.uadd.with.overflow(<16 x i16> %a, <16 x i16> %b)
  %e0 = extractvalue {<16 x i16>, <16 x i1>} %o, 0
  %e1 = extractvalue {<16 x i16>, <16 x i1>} %o, 1
  %r = select <16 x i1> %e1, <16 x i16> %e0, <16 x i16> %a
  ret <16 x i16> %r
}

define <16 x i16> @sadd_v16i16(<16 x i16> %a, <16 x i16> %b) {
; CHECK-SD-LABEL: sadd_v16i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqadd v4.8h, v1.8h, v3.8h
; CHECK-SD-NEXT:    sqadd v5.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    add v3.8h, v1.8h, v3.8h
; CHECK-SD-NEXT:    add v2.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    cmeq v4.8h, v3.8h, v4.8h
; CHECK-SD-NEXT:    cmeq v5.8h, v2.8h, v5.8h
; CHECK-SD-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v16i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v4.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    add v5.8h, v1.8h, v3.8h
; CHECK-GI-NEXT:    cmlt v2.8h, v2.8h, #0
; CHECK-GI-NEXT:    cmlt v3.8h, v3.8h, #0
; CHECK-GI-NEXT:    cmgt v6.8h, v0.8h, v4.8h
; CHECK-GI-NEXT:    cmgt v7.8h, v1.8h, v5.8h
; CHECK-GI-NEXT:    uzp1 v2.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    uzp1 v3.16b, v6.16b, v7.16b
; CHECK-GI-NEXT:    eor v2.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    sshll v3.8h, v2.8b, #0
; CHECK-GI-NEXT:    sshll2 v2.8h, v2.16b, #0
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v3.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<16 x i16>, <16 x i1>} @llvm.sadd.with.overflow(<16 x i16> %a, <16 x i16> %b)
  %e0 = extractvalue {<16 x i16>, <16 x i1>} %o, 0
  %e1 = extractvalue {<16 x i16>, <16 x i1>} %o, 1
  %r = select <16 x i1> %e1, <16 x i16> %e0, <16 x i16> %a
  ret <16 x i16> %r
}

define <16 x i16> @usub_v16i16(<16 x i16> %a, <16 x i16> %b) {
; CHECK-SD-LABEL: usub_v16i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub v3.8h, v1.8h, v3.8h
; CHECK-SD-NEXT:    sub v2.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    cmhi v4.8h, v3.8h, v1.8h
; CHECK-SD-NEXT:    cmhi v5.8h, v2.8h, v0.8h
; CHECK-SD-NEXT:    bit v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bit v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v16i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v4.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    sub v5.8h, v1.8h, v3.8h
; CHECK-GI-NEXT:    cmhi v2.8h, v2.8h, v0.8h
; CHECK-GI-NEXT:    cmhi v3.8h, v3.8h, v1.8h
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v2.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v3.16b
; CHECK-GI-NEXT:    ret
  %o = call {<16 x i16>, <16 x i1>} @llvm.usub.with.overflow(<16 x i16> %a, <16 x i16> %b)
  %e0 = extractvalue {<16 x i16>, <16 x i1>} %o, 0
  %e1 = extractvalue {<16 x i16>, <16 x i1>} %o, 1
  %r = select <16 x i1> %e1, <16 x i16> %e0, <16 x i16> %a
  ret <16 x i16> %r
}

define <16 x i16> @ssub_v16i16(<16 x i16> %a, <16 x i16> %b) {
; CHECK-SD-LABEL: ssub_v16i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqsub v4.8h, v1.8h, v3.8h
; CHECK-SD-NEXT:    sqsub v5.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    sub v3.8h, v1.8h, v3.8h
; CHECK-SD-NEXT:    sub v2.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    cmeq v4.8h, v3.8h, v4.8h
; CHECK-SD-NEXT:    cmeq v5.8h, v2.8h, v5.8h
; CHECK-SD-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v16i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v4.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    sub v5.8h, v1.8h, v3.8h
; CHECK-GI-NEXT:    cmgt v2.8h, v2.8h, v0.8h
; CHECK-GI-NEXT:    cmgt v3.8h, v3.8h, v1.8h
; CHECK-GI-NEXT:    cmlt v6.8h, v4.8h, #0
; CHECK-GI-NEXT:    cmlt v7.8h, v5.8h, #0
; CHECK-GI-NEXT:    uzp1 v2.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    uzp1 v3.16b, v6.16b, v7.16b
; CHECK-GI-NEXT:    eor v2.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    sshll v3.8h, v2.8b, #0
; CHECK-GI-NEXT:    sshll2 v2.8h, v2.16b, #0
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v3.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<16 x i16>, <16 x i1>} @llvm.ssub.with.overflow(<16 x i16> %a, <16 x i16> %b)
  %e0 = extractvalue {<16 x i16>, <16 x i1>} %o, 0
  %e1 = extractvalue {<16 x i16>, <16 x i1>} %o, 1
  %r = select <16 x i1> %e1, <16 x i16> %e0, <16 x i16> %a
  ret <16 x i16> %r
}

define <16 x i16> @umul_v16i16(<16 x i16> %a, <16 x i16> %b) {
; CHECK-SD-LABEL: umul_v16i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    umull2 v4.4s, v1.8h, v3.8h
; CHECK-SD-NEXT:    umull v5.4s, v1.4h, v3.4h
; CHECK-SD-NEXT:    umull2 v6.4s, v0.8h, v2.8h
; CHECK-SD-NEXT:    umull v7.4s, v0.4h, v2.4h
; CHECK-SD-NEXT:    mul v3.8h, v1.8h, v3.8h
; CHECK-SD-NEXT:    mul v2.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    uzp2 v4.8h, v5.8h, v4.8h
; CHECK-SD-NEXT:    uzp2 v5.8h, v7.8h, v6.8h
; CHECK-SD-NEXT:    cmeq v4.8h, v4.8h, #0
; CHECK-SD-NEXT:    cmeq v5.8h, v5.8h, #0
; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v16i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    umull v4.4s, v0.4h, v2.4h
; CHECK-GI-NEXT:    umull2 v5.4s, v0.8h, v2.8h
; CHECK-GI-NEXT:    umull v6.4s, v1.4h, v3.4h
; CHECK-GI-NEXT:    umull2 v7.4s, v1.8h, v3.8h
; CHECK-GI-NEXT:    mul v2.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    mul v3.8h, v1.8h, v3.8h
; CHECK-GI-NEXT:    uzp2 v4.8h, v4.8h, v5.8h
; CHECK-GI-NEXT:    uzp2 v5.8h, v6.8h, v7.8h
; CHECK-GI-NEXT:    cmtst v4.8h, v4.8h, v4.8h
; CHECK-GI-NEXT:    cmtst v5.8h, v5.8h, v5.8h
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    bit v1.16b, v3.16b, v5.16b
; CHECK-GI-NEXT:    ret
  %o = call {<16 x i16>, <16 x i1>} @llvm.umul.with.overflow(<16 x i16> %a, <16 x i16> %b)
  %e0 = extractvalue {<16 x i16>, <16 x i1>} %o, 0
  %e1 = extractvalue {<16 x i16>, <16 x i1>} %o, 1
  %r = select <16 x i1> %e1, <16 x i16> %e0, <16 x i16> %a
  ret <16 x i16> %r
}

define <16 x i16> @smul_v16i16(<16 x i16> %a, <16 x i16> %b) {
; CHECK-SD-LABEL: smul_v16i16:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    smull2 v4.4s, v1.8h, v3.8h
; CHECK-SD-NEXT:    smull v5.4s, v1.4h, v3.4h
; CHECK-SD-NEXT:    smull2 v6.4s, v0.8h, v2.8h
; CHECK-SD-NEXT:    smull v7.4s, v0.4h, v2.4h
; CHECK-SD-NEXT:    mul v3.8h, v1.8h, v3.8h
; CHECK-SD-NEXT:    mul v2.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    uzp2 v4.8h, v5.8h, v4.8h
; CHECK-SD-NEXT:    uzp2 v5.8h, v7.8h, v6.8h
; CHECK-SD-NEXT:    cmlt v6.8h, v3.8h, #0
; CHECK-SD-NEXT:    cmlt v7.8h, v2.8h, #0
; CHECK-SD-NEXT:    cmeq v4.8h, v4.8h, v6.8h
; CHECK-SD-NEXT:    cmeq v5.8h, v5.8h, v7.8h
; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v16i16:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v4.2d, #0xff00ff00ff00ff
; CHECK-GI-NEXT:    smull v5.4s, v0.4h, v2.4h
; CHECK-GI-NEXT:    smull v6.4s, v1.4h, v3.4h
; CHECK-GI-NEXT:    smull2 v7.4s, v0.8h, v2.8h
; CHECK-GI-NEXT:    mul v2.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    smull2 v16.4s, v1.8h, v3.8h
; CHECK-GI-NEXT:    mul v3.8h, v1.8h, v3.8h
; CHECK-GI-NEXT:    neg v4.8h, v4.8h
; CHECK-GI-NEXT:    shrn v5.4h, v5.4s, #16
; CHECK-GI-NEXT:    shrn v6.4h, v6.4s, #16
; CHECK-GI-NEXT:    sshl v17.8h, v2.8h, v4.8h
; CHECK-GI-NEXT:    sshl v4.8h, v3.8h, v4.8h
; CHECK-GI-NEXT:    shrn2 v5.8h, v7.4s, #16
; CHECK-GI-NEXT:    shrn2 v6.8h, v16.4s, #16
; CHECK-GI-NEXT:    cmeq v5.8h, v5.8h, v17.8h
; CHECK-GI-NEXT:    cmeq v4.8h, v6.8h, v4.8h
; CHECK-GI-NEXT:    mvn v5.16b, v5.16b
; CHECK-GI-NEXT:    mvn v4.16b, v4.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v5.16b
; CHECK-GI-NEXT:    bit v1.16b, v3.16b, v4.16b
; CHECK-GI-NEXT:    ret
  %o = call {<16 x i16>, <16 x i1>} @llvm.smul.with.overflow(<16 x i16> %a, <16 x i16> %b)
  %e0 = extractvalue {<16 x i16>, <16 x i1>} %o, 0
  %e1 = extractvalue {<16 x i16>, <16 x i1>} %o, 1
  %r = select <16 x i1> %e1, <16 x i16> %e0, <16 x i16> %a
  ret <16 x i16> %r
}

define <2 x i32> @uadd_v2i32(<2 x i32> %a, <2 x i32> %b) {
; CHECK-SD-LABEL: uadd_v2i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    add v1.2s, v0.2s, v1.2s
; CHECK-SD-NEXT:    cmhi v2.2s, v0.2s, v1.2s
; CHECK-SD-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v2i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    cmhi v1.2s, v1.2s, v2.2s
; CHECK-GI-NEXT:    bit v0.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i32>, <2 x i1>} @llvm.uadd.with.overflow(<2 x i32> %a, <2 x i32> %b)
  %e0 = extractvalue {<2 x i32>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i32>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i32> %e0, <2 x i32> %a
  ret <2 x i32> %r
}

define <2 x i32> @sadd_v2i32(<2 x i32> %a, <2 x i32> %b) {
; CHECK-SD-LABEL: sadd_v2i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqadd v2.2s, v0.2s, v1.2s
; CHECK-SD-NEXT:    add v1.2s, v0.2s, v1.2s
; CHECK-SD-NEXT:    cmeq v2.2s, v1.2s, v2.2s
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v2i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    cmlt v1.2s, v1.2s, #0
; CHECK-GI-NEXT:    cmgt v3.2s, v0.2s, v2.2s
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    bit v0.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i32>, <2 x i1>} @llvm.sadd.with.overflow(<2 x i32> %a, <2 x i32> %b)
  %e0 = extractvalue {<2 x i32>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i32>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i32> %e0, <2 x i32> %a
  ret <2 x i32> %r
}

define <2 x i32> @usub_v2i32(<2 x i32> %a, <2 x i32> %b) {
; CHECK-SD-LABEL: usub_v2i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub v1.2s, v0.2s, v1.2s
; CHECK-SD-NEXT:    cmhi v2.2s, v1.2s, v0.2s
; CHECK-SD-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v2i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    cmhi v1.2s, v1.2s, v0.2s
; CHECK-GI-NEXT:    bit v0.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i32>, <2 x i1>} @llvm.usub.with.overflow(<2 x i32> %a, <2 x i32> %b)
  %e0 = extractvalue {<2 x i32>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i32>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i32> %e0, <2 x i32> %a
  ret <2 x i32> %r
}

define <2 x i32> @ssub_v2i32(<2 x i32> %a, <2 x i32> %b) {
; CHECK-SD-LABEL: ssub_v2i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqsub v2.2s, v0.2s, v1.2s
; CHECK-SD-NEXT:    sub v1.2s, v0.2s, v1.2s
; CHECK-SD-NEXT:    cmeq v2.2s, v1.2s, v2.2s
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v2i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    cmgt v1.2s, v1.2s, v0.2s
; CHECK-GI-NEXT:    cmlt v3.2s, v2.2s, #0
; CHECK-GI-NEXT:    eor v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    bit v0.8b, v2.8b, v1.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i32>, <2 x i1>} @llvm.ssub.with.overflow(<2 x i32> %a, <2 x i32> %b)
  %e0 = extractvalue {<2 x i32>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i32>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i32> %e0, <2 x i32> %a
  ret <2 x i32> %r
}

define <2 x i32> @umul_v2i32(<2 x i32> %a, <2 x i32> %b) {
; CHECK-SD-LABEL: umul_v2i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v1.2s
; CHECK-SD-NEXT:    shrn v2.2s, v1.2d, #32
; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
; CHECK-SD-NEXT:    cmeq v2.2s, v2.2s, #0
; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v2i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    umull v2.2d, v0.2s, v1.2s
; CHECK-GI-NEXT:    mul v1.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    shrn v2.2s, v2.2d, #32
; CHECK-GI-NEXT:    cmtst v2.2s, v2.2s, v2.2s
; CHECK-GI-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow(<2 x i32> %a, <2 x i32> %b)
  %e0 = extractvalue {<2 x i32>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i32>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i32> %e0, <2 x i32> %a
  ret <2 x i32> %r
}

define <2 x i32> @smul_v2i32(<2 x i32> %a, <2 x i32> %b) {
; CHECK-SD-LABEL: smul_v2i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
; CHECK-SD-NEXT:    xtn v2.2s, v1.2d
; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
; CHECK-SD-NEXT:    cmlt v3.2s, v2.2s, #0
; CHECK-SD-NEXT:    cmeq v1.2s, v1.2s, v3.2s
; CHECK-SD-NEXT:    bif v0.8b, v2.8b, v1.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v2i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v2.2s, #63
; CHECK-GI-NEXT:    smull v3.2d, v0.2s, v1.2s
; CHECK-GI-NEXT:    mul v1.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    neg v2.2s, v2.2s
; CHECK-GI-NEXT:    shrn v3.2s, v3.2d, #32
; CHECK-GI-NEXT:    sshl v2.2s, v1.2s, v2.2s
; CHECK-GI-NEXT:    cmeq v2.2s, v3.2s, v2.2s
; CHECK-GI-NEXT:    mvn v2.8b, v2.8b
; CHECK-GI-NEXT:    bit v0.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i32>, <2 x i1>} @llvm.smul.with.overflow(<2 x i32> %a, <2 x i32> %b)
  %e0 = extractvalue {<2 x i32>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i32>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i32> %e0, <2 x i32> %a
  ret <2 x i32> %r
}

define <3 x i32> @uadd_v3i32(<3 x i32> %a, <3 x i32> %b) {
; CHECK-SD-LABEL: uadd_v3i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    add v1.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    cmhi v2.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v3i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v3.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    movi v2.2d, #0xffffffffffffffff
; CHECK-GI-NEXT:    cmhi v1.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    eor v2.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    and v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i32>, <3 x i1>} @llvm.uadd.with.overflow(<3 x i32> %a, <3 x i32> %b)
  %e0 = extractvalue {<3 x i32>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i32>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i32> %e0, <3 x i32> %a
  ret <3 x i32> %r
}

define <3 x i32> @sadd_v3i32(<3 x i32> %a, <3 x i32> %b) {
; CHECK-SD-LABEL: sadd_v3i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqadd v2.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    add v1.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    cmeq v2.4s, v1.4s, v2.4s
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v3i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    add v3.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    cmgt v4.4s, v0.4s, v3.4s
; CHECK-GI-NEXT:    cmgt v1.4s, v2.4s, v1.4s
; CHECK-GI-NEXT:    movi v2.2d, #0xffffffffffffffff
; CHECK-GI-NEXT:    eor v1.16b, v1.16b, v4.16b
; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #31
; CHECK-GI-NEXT:    cmlt v1.4s, v1.4s, #0
; CHECK-GI-NEXT:    eor v2.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    and v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i32>, <3 x i1>} @llvm.sadd.with.overflow(<3 x i32> %a, <3 x i32> %b)
  %e0 = extractvalue {<3 x i32>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i32>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i32> %e0, <3 x i32> %a
  ret <3 x i32> %r
}

define <3 x i32> @usub_v3i32(<3 x i32> %a, <3 x i32> %b) {
; CHECK-SD-LABEL: usub_v3i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub v1.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    cmhi v2.4s, v1.4s, v0.4s
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v3i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v2.2d, #0xffffffffffffffff
; CHECK-GI-NEXT:    cmhi v3.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    sub v1.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    and v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    eor v2.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i32>, <3 x i1>} @llvm.usub.with.overflow(<3 x i32> %a, <3 x i32> %b)
  %e0 = extractvalue {<3 x i32>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i32>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i32> %e0, <3 x i32> %a
  ret <3 x i32> %r
}

define <3 x i32> @ssub_v3i32(<3 x i32> %a, <3 x i32> %b) {
; CHECK-SD-LABEL: ssub_v3i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqsub v2.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    sub v1.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    cmeq v2.4s, v1.4s, v2.4s
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v3i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    sub v3.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    cmgt v1.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    cmgt v2.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    eor v1.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    movi v2.2d, #0xffffffffffffffff
; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #31
; CHECK-GI-NEXT:    cmlt v1.4s, v1.4s, #0
; CHECK-GI-NEXT:    eor v2.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    and v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    and v0.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i32>, <3 x i1>} @llvm.ssub.with.overflow(<3 x i32> %a, <3 x i32> %b)
  %e0 = extractvalue {<3 x i32>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i32>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i32> %e0, <3 x i32> %a
  ret <3 x i32> %r
}

define <3 x i32> @umul_v3i32(<3 x i32> %a, <3 x i32> %b) {
; CHECK-SD-LABEL: umul_v3i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    umull2 v2.2d, v0.4s, v1.4s
; CHECK-SD-NEXT:    umull v3.2d, v0.2s, v1.2s
; CHECK-SD-NEXT:    mul v1.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    uzp2 v2.4s, v3.4s, v2.4s
; CHECK-SD-NEXT:    cmeq v2.4s, v2.4s, #0
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v3i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    mov v3.s[0], v0.s[0]
; CHECK-GI-NEXT:    mov v4.s[0], v1.s[0]
; CHECK-GI-NEXT:    mov w8, v0.s[2]
; CHECK-GI-NEXT:    mov w9, v1.s[2]
; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
; CHECK-GI-NEXT:    mov v3.s[1], v0.s[1]
; CHECK-GI-NEXT:    mov v4.s[1], v1.s[1]
; CHECK-GI-NEXT:    mul v1.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    umull x8, w8, w9
; CHECK-GI-NEXT:    umull v3.2d, v3.2s, v4.2s
; CHECK-GI-NEXT:    lsr x8, x8, #32
; CHECK-GI-NEXT:    ushr v3.2d, v3.2d, #32
; CHECK-GI-NEXT:    mov d4, v3.d[1]
; CHECK-GI-NEXT:    fmov x10, d3
; CHECK-GI-NEXT:    fmov s3, w10
; CHECK-GI-NEXT:    fmov x9, d4
; CHECK-GI-NEXT:    mov v3.s[1], w9
; CHECK-GI-NEXT:    mov v3.s[2], w8
; CHECK-GI-NEXT:    cmeq v2.4s, v3.4s, v2.4s
; CHECK-GI-NEXT:    movi v3.2d, #0xffffffffffffffff
; CHECK-GI-NEXT:    mvn v4.16b, v2.16b
; CHECK-GI-NEXT:    bic v1.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    eor v3.16b, v4.16b, v3.16b
; CHECK-GI-NEXT:    and v0.16b, v0.16b, v3.16b
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow(<3 x i32> %a, <3 x i32> %b)
  %e0 = extractvalue {<3 x i32>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i32>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i32> %e0, <3 x i32> %a
  ret <3 x i32> %r
}

define <3 x i32> @smul_v3i32(<3 x i32> %a, <3 x i32> %b) {
; CHECK-SD-LABEL: smul_v3i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    smull2 v2.2d, v0.4s, v1.4s
; CHECK-SD-NEXT:    smull v3.2d, v0.2s, v1.2s
; CHECK-SD-NEXT:    mul v1.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    uzp2 v2.4s, v3.4s, v2.4s
; CHECK-SD-NEXT:    cmlt v3.4s, v1.4s, #0
; CHECK-SD-NEXT:    cmeq v2.4s, v2.4s, v3.4s
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v3i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    mov v3.s[0], v0.s[0]
; CHECK-GI-NEXT:    mov v4.s[0], v1.s[0]
; CHECK-GI-NEXT:    mov w8, v0.s[2]
; CHECK-GI-NEXT:    mov w9, v1.s[2]
; CHECK-GI-NEXT:    movi v2.4s, #95
; CHECK-GI-NEXT:    mov v3.s[1], v0.s[1]
; CHECK-GI-NEXT:    mov v4.s[1], v1.s[1]
; CHECK-GI-NEXT:    mul v1.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    smull x8, w8, w9
; CHECK-GI-NEXT:    neg v2.4s, v2.4s
; CHECK-GI-NEXT:    smull v3.2d, v3.2s, v4.2s
; CHECK-GI-NEXT:    asr x8, x8, #32
; CHECK-GI-NEXT:    sshl v2.4s, v1.4s, v2.4s
; CHECK-GI-NEXT:    sshr v3.2d, v3.2d, #32
; CHECK-GI-NEXT:    mov d4, v3.d[1]
; CHECK-GI-NEXT:    fmov x10, d3
; CHECK-GI-NEXT:    fmov s3, w10
; CHECK-GI-NEXT:    fmov x9, d4
; CHECK-GI-NEXT:    mov v3.s[1], w9
; CHECK-GI-NEXT:    mov v3.s[2], w8
; CHECK-GI-NEXT:    cmeq v2.4s, v3.4s, v2.4s
; CHECK-GI-NEXT:    movi v3.2d, #0xffffffffffffffff
; CHECK-GI-NEXT:    mvn v4.16b, v2.16b
; CHECK-GI-NEXT:    bic v1.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    eor v3.16b, v4.16b, v3.16b
; CHECK-GI-NEXT:    and v0.16b, v0.16b, v3.16b
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i32>, <3 x i1>} @llvm.smul.with.overflow(<3 x i32> %a, <3 x i32> %b)
  %e0 = extractvalue {<3 x i32>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i32>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i32> %e0, <3 x i32> %a
  ret <3 x i32> %r
}

define <4 x i32> @uadd_v4i32(<4 x i32> %a, <4 x i32> %b) {
; CHECK-SD-LABEL: uadd_v4i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    add v1.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    cmhi v2.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v4i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    cmhi v1.4s, v1.4s, v2.4s
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i32>, <4 x i1>} @llvm.uadd.with.overflow(<4 x i32> %a, <4 x i32> %b)
  %e0 = extractvalue {<4 x i32>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i32>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i32> %e0, <4 x i32> %a
  ret <4 x i32> %r
}

define <4 x i32> @sadd_v4i32(<4 x i32> %a, <4 x i32> %b) {
; CHECK-SD-LABEL: sadd_v4i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqadd v2.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    add v1.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    cmeq v2.4s, v1.4s, v2.4s
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v4i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    cmlt v1.4s, v1.4s, #0
; CHECK-GI-NEXT:    cmgt v3.4s, v0.4s, v2.4s
; CHECK-GI-NEXT:    eor v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i32>, <4 x i1>} @llvm.sadd.with.overflow(<4 x i32> %a, <4 x i32> %b)
  %e0 = extractvalue {<4 x i32>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i32>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i32> %e0, <4 x i32> %a
  ret <4 x i32> %r
}

define <4 x i32> @usub_v4i32(<4 x i32> %a, <4 x i32> %b) {
; CHECK-SD-LABEL: usub_v4i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub v1.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    cmhi v2.4s, v1.4s, v0.4s
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v4i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    cmhi v1.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i32>, <4 x i1>} @llvm.usub.with.overflow(<4 x i32> %a, <4 x i32> %b)
  %e0 = extractvalue {<4 x i32>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i32>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i32> %e0, <4 x i32> %a
  ret <4 x i32> %r
}

define <4 x i32> @ssub_v4i32(<4 x i32> %a, <4 x i32> %b) {
; CHECK-SD-LABEL: ssub_v4i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqsub v2.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    sub v1.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    cmeq v2.4s, v1.4s, v2.4s
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v4i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    cmgt v1.4s, v1.4s, v0.4s
; CHECK-GI-NEXT:    cmlt v3.4s, v2.4s, #0
; CHECK-GI-NEXT:    eor v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i32>, <4 x i1>} @llvm.ssub.with.overflow(<4 x i32> %a, <4 x i32> %b)
  %e0 = extractvalue {<4 x i32>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i32>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i32> %e0, <4 x i32> %a
  ret <4 x i32> %r
}

define <4 x i32> @umul_v4i32(<4 x i32> %a, <4 x i32> %b) {
; CHECK-SD-LABEL: umul_v4i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    umull2 v2.2d, v0.4s, v1.4s
; CHECK-SD-NEXT:    umull v3.2d, v0.2s, v1.2s
; CHECK-SD-NEXT:    mul v1.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    uzp2 v2.4s, v3.4s, v2.4s
; CHECK-SD-NEXT:    cmeq v2.4s, v2.4s, #0
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v4i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    umull2 v2.2d, v0.4s, v1.4s
; CHECK-GI-NEXT:    umull v3.2d, v0.2s, v1.2s
; CHECK-GI-NEXT:    mul v1.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    uzp2 v2.4s, v3.4s, v2.4s
; CHECK-GI-NEXT:    cmtst v2.4s, v2.4s, v2.4s
; CHECK-GI-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow(<4 x i32> %a, <4 x i32> %b)
  %e0 = extractvalue {<4 x i32>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i32>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i32> %e0, <4 x i32> %a
  ret <4 x i32> %r
}

define <4 x i32> @smul_v4i32(<4 x i32> %a, <4 x i32> %b) {
; CHECK-SD-LABEL: smul_v4i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    smull2 v2.2d, v0.4s, v1.4s
; CHECK-SD-NEXT:    smull v3.2d, v0.2s, v1.2s
; CHECK-SD-NEXT:    mul v1.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    uzp2 v2.4s, v3.4s, v2.4s
; CHECK-SD-NEXT:    cmlt v3.4s, v1.4s, #0
; CHECK-SD-NEXT:    cmeq v2.4s, v2.4s, v3.4s
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v4i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v2.4s, #127
; CHECK-GI-NEXT:    smull2 v3.2d, v0.4s, v1.4s
; CHECK-GI-NEXT:    smull v4.2d, v0.2s, v1.2s
; CHECK-GI-NEXT:    mul v1.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    neg v2.4s, v2.4s
; CHECK-GI-NEXT:    uzp2 v3.4s, v4.4s, v3.4s
; CHECK-GI-NEXT:    sshl v2.4s, v1.4s, v2.4s
; CHECK-GI-NEXT:    cmeq v2.4s, v3.4s, v2.4s
; CHECK-GI-NEXT:    mvn v2.16b, v2.16b
; CHECK-GI-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i32>, <4 x i1>} @llvm.smul.with.overflow(<4 x i32> %a, <4 x i32> %b)
  %e0 = extractvalue {<4 x i32>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i32>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i32> %e0, <4 x i32> %a
  ret <4 x i32> %r
}

define <8 x i32> @uadd_v8i32(<8 x i32> %a, <8 x i32> %b) {
; CHECK-SD-LABEL: uadd_v8i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    add v3.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    add v2.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    cmhi v4.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    cmhi v5.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    bit v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bit v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v8i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v4.4s, v0.4s, v2.4s
; CHECK-GI-NEXT:    add v5.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    cmhi v2.4s, v2.4s, v4.4s
; CHECK-GI-NEXT:    cmhi v3.4s, v3.4s, v5.4s
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v2.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v3.16b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i32>, <8 x i1>} @llvm.uadd.with.overflow(<8 x i32> %a, <8 x i32> %b)
  %e0 = extractvalue {<8 x i32>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i32>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i32> %e0, <8 x i32> %a
  ret <8 x i32> %r
}

define <8 x i32> @sadd_v8i32(<8 x i32> %a, <8 x i32> %b) {
; CHECK-SD-LABEL: sadd_v8i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqadd v4.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    sqadd v5.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    add v3.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    add v2.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    cmeq v4.4s, v3.4s, v4.4s
; CHECK-SD-NEXT:    cmeq v5.4s, v2.4s, v5.4s
; CHECK-SD-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v8i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v4.4s, v0.4s, v2.4s
; CHECK-GI-NEXT:    add v5.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    cmlt v2.4s, v2.4s, #0
; CHECK-GI-NEXT:    cmlt v3.4s, v3.4s, #0
; CHECK-GI-NEXT:    cmgt v6.4s, v0.4s, v4.4s
; CHECK-GI-NEXT:    cmgt v7.4s, v1.4s, v5.4s
; CHECK-GI-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
; CHECK-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
; CHECK-GI-NEXT:    eor v2.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    sshll v3.4s, v2.4h, #0
; CHECK-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v3.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i32>, <8 x i1>} @llvm.sadd.with.overflow(<8 x i32> %a, <8 x i32> %b)
  %e0 = extractvalue {<8 x i32>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i32>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i32> %e0, <8 x i32> %a
  ret <8 x i32> %r
}

define <8 x i32> @usub_v8i32(<8 x i32> %a, <8 x i32> %b) {
; CHECK-SD-LABEL: usub_v8i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub v3.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    sub v2.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    cmhi v4.4s, v3.4s, v1.4s
; CHECK-SD-NEXT:    cmhi v5.4s, v2.4s, v0.4s
; CHECK-SD-NEXT:    bit v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bit v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v8i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v4.4s, v0.4s, v2.4s
; CHECK-GI-NEXT:    sub v5.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    cmhi v2.4s, v2.4s, v0.4s
; CHECK-GI-NEXT:    cmhi v3.4s, v3.4s, v1.4s
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v2.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v3.16b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i32>, <8 x i1>} @llvm.usub.with.overflow(<8 x i32> %a, <8 x i32> %b)
  %e0 = extractvalue {<8 x i32>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i32>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i32> %e0, <8 x i32> %a
  ret <8 x i32> %r
}

define <8 x i32> @ssub_v8i32(<8 x i32> %a, <8 x i32> %b) {
; CHECK-SD-LABEL: ssub_v8i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqsub v4.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    sqsub v5.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    sub v3.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    sub v2.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    cmeq v4.4s, v3.4s, v4.4s
; CHECK-SD-NEXT:    cmeq v5.4s, v2.4s, v5.4s
; CHECK-SD-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v8i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v4.4s, v0.4s, v2.4s
; CHECK-GI-NEXT:    sub v5.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    cmgt v2.4s, v2.4s, v0.4s
; CHECK-GI-NEXT:    cmgt v3.4s, v3.4s, v1.4s
; CHECK-GI-NEXT:    cmlt v6.4s, v4.4s, #0
; CHECK-GI-NEXT:    cmlt v7.4s, v5.4s, #0
; CHECK-GI-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
; CHECK-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
; CHECK-GI-NEXT:    eor v2.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    sshll v3.4s, v2.4h, #0
; CHECK-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v3.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i32>, <8 x i1>} @llvm.ssub.with.overflow(<8 x i32> %a, <8 x i32> %b)
  %e0 = extractvalue {<8 x i32>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i32>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i32> %e0, <8 x i32> %a
  ret <8 x i32> %r
}

define <8 x i32> @umul_v8i32(<8 x i32> %a, <8 x i32> %b) {
; CHECK-SD-LABEL: umul_v8i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    umull2 v4.2d, v1.4s, v3.4s
; CHECK-SD-NEXT:    umull v5.2d, v1.2s, v3.2s
; CHECK-SD-NEXT:    umull2 v6.2d, v0.4s, v2.4s
; CHECK-SD-NEXT:    umull v7.2d, v0.2s, v2.2s
; CHECK-SD-NEXT:    mul v3.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    mul v2.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    uzp2 v4.4s, v5.4s, v4.4s
; CHECK-SD-NEXT:    uzp2 v5.4s, v7.4s, v6.4s
; CHECK-SD-NEXT:    cmeq v4.4s, v4.4s, #0
; CHECK-SD-NEXT:    cmeq v5.4s, v5.4s, #0
; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v8i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    umull v4.2d, v0.2s, v2.2s
; CHECK-GI-NEXT:    umull2 v5.2d, v0.4s, v2.4s
; CHECK-GI-NEXT:    umull v6.2d, v1.2s, v3.2s
; CHECK-GI-NEXT:    umull2 v7.2d, v1.4s, v3.4s
; CHECK-GI-NEXT:    mul v2.4s, v0.4s, v2.4s
; CHECK-GI-NEXT:    mul v3.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    uzp2 v4.4s, v4.4s, v5.4s
; CHECK-GI-NEXT:    uzp2 v5.4s, v6.4s, v7.4s
; CHECK-GI-NEXT:    cmtst v4.4s, v4.4s, v4.4s
; CHECK-GI-NEXT:    cmtst v5.4s, v5.4s, v5.4s
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    bit v1.16b, v3.16b, v5.16b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow(<8 x i32> %a, <8 x i32> %b)
  %e0 = extractvalue {<8 x i32>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i32>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i32> %e0, <8 x i32> %a
  ret <8 x i32> %r
}

define <8 x i32> @smul_v8i32(<8 x i32> %a, <8 x i32> %b) {
; CHECK-SD-LABEL: smul_v8i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    smull2 v4.2d, v1.4s, v3.4s
; CHECK-SD-NEXT:    smull v5.2d, v1.2s, v3.2s
; CHECK-SD-NEXT:    smull2 v6.2d, v0.4s, v2.4s
; CHECK-SD-NEXT:    smull v7.2d, v0.2s, v2.2s
; CHECK-SD-NEXT:    mul v3.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    mul v2.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    uzp2 v4.4s, v5.4s, v4.4s
; CHECK-SD-NEXT:    uzp2 v5.4s, v7.4s, v6.4s
; CHECK-SD-NEXT:    cmlt v6.4s, v3.4s, #0
; CHECK-SD-NEXT:    cmlt v7.4s, v2.4s, #0
; CHECK-SD-NEXT:    cmeq v4.4s, v4.4s, v6.4s
; CHECK-SD-NEXT:    cmeq v5.4s, v5.4s, v7.4s
; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v8i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    movi v4.2d, #0x0000ff000000ff
; CHECK-GI-NEXT:    smull v5.2d, v0.2s, v2.2s
; CHECK-GI-NEXT:    smull v6.2d, v1.2s, v3.2s
; CHECK-GI-NEXT:    smull2 v7.2d, v0.4s, v2.4s
; CHECK-GI-NEXT:    mul v2.4s, v0.4s, v2.4s
; CHECK-GI-NEXT:    smull2 v16.2d, v1.4s, v3.4s
; CHECK-GI-NEXT:    mul v3.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    neg v4.4s, v4.4s
; CHECK-GI-NEXT:    shrn v5.2s, v5.2d, #32
; CHECK-GI-NEXT:    shrn v6.2s, v6.2d, #32
; CHECK-GI-NEXT:    sshl v17.4s, v2.4s, v4.4s
; CHECK-GI-NEXT:    sshl v4.4s, v3.4s, v4.4s
; CHECK-GI-NEXT:    shrn2 v5.4s, v7.2d, #32
; CHECK-GI-NEXT:    shrn2 v6.4s, v16.2d, #32
; CHECK-GI-NEXT:    cmeq v5.4s, v5.4s, v17.4s
; CHECK-GI-NEXT:    cmeq v4.4s, v6.4s, v4.4s
; CHECK-GI-NEXT:    mvn v5.16b, v5.16b
; CHECK-GI-NEXT:    mvn v4.16b, v4.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v5.16b
; CHECK-GI-NEXT:    bit v1.16b, v3.16b, v4.16b
; CHECK-GI-NEXT:    ret
  %o = call {<8 x i32>, <8 x i1>} @llvm.smul.with.overflow(<8 x i32> %a, <8 x i32> %b)
  %e0 = extractvalue {<8 x i32>, <8 x i1>} %o, 0
  %e1 = extractvalue {<8 x i32>, <8 x i1>} %o, 1
  %r = select <8 x i1> %e1, <8 x i32> %e0, <8 x i32> %a
  ret <8 x i32> %r
}

define <2 x i64> @uadd_v2i64(<2 x i64> %a, <2 x i64> %b) {
; CHECK-SD-LABEL: uadd_v2i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    add v1.2d, v0.2d, v1.2d
; CHECK-SD-NEXT:    cmhi v2.2d, v0.2d, v1.2d
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v2i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.2d, v0.2d, v1.2d
; CHECK-GI-NEXT:    cmhi v1.2d, v1.2d, v2.2d
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow(<2 x i64> %a, <2 x i64> %b)
  %e0 = extractvalue {<2 x i64>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i64>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i64> %e0, <2 x i64> %a
  ret <2 x i64> %r
}

define <2 x i64> @sadd_v2i64(<2 x i64> %a, <2 x i64> %b) {
; CHECK-SD-LABEL: sadd_v2i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqadd v2.2d, v0.2d, v1.2d
; CHECK-SD-NEXT:    add v1.2d, v0.2d, v1.2d
; CHECK-SD-NEXT:    cmeq v2.2d, v1.2d, v2.2d
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v2i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v2.2d, v0.2d, v1.2d
; CHECK-GI-NEXT:    cmlt v1.2d, v1.2d, #0
; CHECK-GI-NEXT:    cmgt v3.2d, v0.2d, v2.2d
; CHECK-GI-NEXT:    eor v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i64>, <2 x i1>} @llvm.sadd.with.overflow(<2 x i64> %a, <2 x i64> %b)
  %e0 = extractvalue {<2 x i64>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i64>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i64> %e0, <2 x i64> %a
  ret <2 x i64> %r
}

define <2 x i64> @usub_v2i64(<2 x i64> %a, <2 x i64> %b) {
; CHECK-SD-LABEL: usub_v2i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub v1.2d, v0.2d, v1.2d
; CHECK-SD-NEXT:    cmhi v2.2d, v1.2d, v0.2d
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v2i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.2d, v0.2d, v1.2d
; CHECK-GI-NEXT:    cmhi v1.2d, v1.2d, v0.2d
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i64>, <2 x i1>} @llvm.usub.with.overflow(<2 x i64> %a, <2 x i64> %b)
  %e0 = extractvalue {<2 x i64>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i64>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i64> %e0, <2 x i64> %a
  ret <2 x i64> %r
}

define <2 x i64> @ssub_v2i64(<2 x i64> %a, <2 x i64> %b) {
; CHECK-SD-LABEL: ssub_v2i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqsub v2.2d, v0.2d, v1.2d
; CHECK-SD-NEXT:    sub v1.2d, v0.2d, v1.2d
; CHECK-SD-NEXT:    cmeq v2.2d, v1.2d, v2.2d
; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v2i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v2.2d, v0.2d, v1.2d
; CHECK-GI-NEXT:    cmgt v1.2d, v1.2d, v0.2d
; CHECK-GI-NEXT:    cmlt v3.2d, v2.2d, #0
; CHECK-GI-NEXT:    eor v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i64>, <2 x i1>} @llvm.ssub.with.overflow(<2 x i64> %a, <2 x i64> %b)
  %e0 = extractvalue {<2 x i64>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i64>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i64> %e0, <2 x i64> %a
  ret <2 x i64> %r
}

define <2 x i64> @umul_v2i64(<2 x i64> %a, <2 x i64> %b) {
; CHECK-SD-LABEL: umul_v2i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov x8, v1.d[1]
; CHECK-SD-NEXT:    mov x9, v0.d[1]
; CHECK-SD-NEXT:    fmov x11, d1
; CHECK-SD-NEXT:    fmov x12, d0
; CHECK-SD-NEXT:    umulh x10, x9, x8
; CHECK-SD-NEXT:    umulh x13, x12, x11
; CHECK-SD-NEXT:    mul x11, x12, x11
; CHECK-SD-NEXT:    cmp xzr, x10
; CHECK-SD-NEXT:    mul x8, x9, x8
; CHECK-SD-NEXT:    csetm x9, ne
; CHECK-SD-NEXT:    cmp xzr, x13
; CHECK-SD-NEXT:    csetm x10, ne
; CHECK-SD-NEXT:    fmov d1, x11
; CHECK-SD-NEXT:    fmov d2, x10
; CHECK-SD-NEXT:    mov v1.d[1], x8
; CHECK-SD-NEXT:    mov v2.d[1], x9
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v2i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    fmov x10, d0
; CHECK-GI-NEXT:    fmov x11, d1
; CHECK-GI-NEXT:    mov x8, v0.d[1]
; CHECK-GI-NEXT:    mov x9, v1.d[1]
; CHECK-GI-NEXT:    umulh x12, x10, x11
; CHECK-GI-NEXT:    umulh x13, x8, x9
; CHECK-GI-NEXT:    mul x10, x10, x11
; CHECK-GI-NEXT:    fmov d1, x12
; CHECK-GI-NEXT:    mul x8, x8, x9
; CHECK-GI-NEXT:    mov v1.d[1], x13
; CHECK-GI-NEXT:    fmov d2, x10
; CHECK-GI-NEXT:    mov v2.d[1], x8
; CHECK-GI-NEXT:    cmtst v1.2d, v1.2d, v1.2d
; CHECK-GI-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow(<2 x i64> %a, <2 x i64> %b)
  %e0 = extractvalue {<2 x i64>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i64>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i64> %e0, <2 x i64> %a
  ret <2 x i64> %r
}

define <2 x i64> @smul_v2i64(<2 x i64> %a, <2 x i64> %b) {
; CHECK-SD-LABEL: smul_v2i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov x8, v1.d[1]
; CHECK-SD-NEXT:    mov x9, v0.d[1]
; CHECK-SD-NEXT:    fmov x11, d0
; CHECK-SD-NEXT:    mul x10, x9, x8
; CHECK-SD-NEXT:    smulh x8, x9, x8
; CHECK-SD-NEXT:    fmov x9, d1
; CHECK-SD-NEXT:    mul x12, x11, x9
; CHECK-SD-NEXT:    smulh x9, x11, x9
; CHECK-SD-NEXT:    cmp x8, x10, asr #63
; CHECK-SD-NEXT:    csetm x8, ne
; CHECK-SD-NEXT:    fmov d2, x12
; CHECK-SD-NEXT:    cmp x9, x12, asr #63
; CHECK-SD-NEXT:    csetm x9, ne
; CHECK-SD-NEXT:    mov v2.d[1], x10
; CHECK-SD-NEXT:    fmov d1, x9
; CHECK-SD-NEXT:    mov v1.d[1], x8
; CHECK-SD-NEXT:    bit v0.16b, v2.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v2i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    fmov x9, d0
; CHECK-GI-NEXT:    fmov x10, d1
; CHECK-GI-NEXT:    adrp x8, .LCPI95_0
; CHECK-GI-NEXT:    mov d2, v1.d[1]
; CHECK-GI-NEXT:    mov d3, v0.d[1]
; CHECK-GI-NEXT:    mov x13, v1.d[1]
; CHECK-GI-NEXT:    mov x14, v0.d[1]
; CHECK-GI-NEXT:    umulh x11, x9, x10
; CHECK-GI-NEXT:    asr x12, x10, #63
; CHECK-GI-NEXT:    fmov x15, d2
; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI95_0]
; CHECK-GI-NEXT:    madd x11, x9, x12, x11
; CHECK-GI-NEXT:    fmov x12, d3
; CHECK-GI-NEXT:    neg v2.2d, v2.2d
; CHECK-GI-NEXT:    mul x13, x14, x13
; CHECK-GI-NEXT:    asr x14, x15, #63
; CHECK-GI-NEXT:    umulh x16, x12, x15
; CHECK-GI-NEXT:    madd x14, x12, x14, x16
; CHECK-GI-NEXT:    asr x16, x9, #63
; CHECK-GI-NEXT:    asr x12, x12, #63
; CHECK-GI-NEXT:    mul x9, x9, x10
; CHECK-GI-NEXT:    madd x10, x16, x10, x11
; CHECK-GI-NEXT:    madd x11, x12, x15, x14
; CHECK-GI-NEXT:    fmov d1, x9
; CHECK-GI-NEXT:    fmov d3, x10
; CHECK-GI-NEXT:    mov v1.d[1], x13
; CHECK-GI-NEXT:    mov v3.d[1], x11
; CHECK-GI-NEXT:    sshl v2.2d, v1.2d, v2.2d
; CHECK-GI-NEXT:    cmeq v2.2d, v3.2d, v2.2d
; CHECK-GI-NEXT:    mvn v2.16b, v2.16b
; CHECK-GI-NEXT:    bit v0.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow(<2 x i64> %a, <2 x i64> %b)
  %e0 = extractvalue {<2 x i64>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i64>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i64> %e0, <2 x i64> %a
  ret <2 x i64> %r
}

define <3 x i64> @uadd_v3i64(<3 x i64> %a, <3 x i64> %b) {
; CHECK-SD-LABEL: uadd_v3i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-SD-NEXT:    mov v3.d[1], v4.d[0]
; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-SD-NEXT:    add d4, d2, d5
; CHECK-SD-NEXT:    add v1.2d, v0.2d, v3.2d
; CHECK-SD-NEXT:    cmhi v3.2d, v0.2d, v1.2d
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    cmhi v1.2d, v2.2d, v4.2d
; CHECK-SD-NEXT:    bit v2.16b, v4.16b, v1.16b
; CHECK-SD-NEXT:    mov d1, v0.d[1]
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v3i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-GI-NEXT:    mov v6.16b, v0.16b
; CHECK-GI-NEXT:    mov v7.16b, v3.16b
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
; CHECK-GI-NEXT:    // kill: def $d5 killed $d5 def $q5
; CHECK-GI-NEXT:    fmov x8, d2
; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-GI-NEXT:    fmov x9, d5
; CHECK-GI-NEXT:    mov v6.d[1], v1.d[0]
; CHECK-GI-NEXT:    mov v7.d[1], v4.d[0]
; CHECK-GI-NEXT:    add x9, x8, x9
; CHECK-GI-NEXT:    add v4.2d, v6.2d, v7.2d
; CHECK-GI-NEXT:    cmhi v1.2d, v3.2d, v4.2d
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v1.16b
; CHECK-GI-NEXT:    fmov d1, x9
; CHECK-GI-NEXT:    cmhi v2.2d, v5.2d, v1.2d
; CHECK-GI-NEXT:    mov d1, v0.d[1]
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT:    fmov x10, d2
; CHECK-GI-NEXT:    and x9, x9, x10
; CHECK-GI-NEXT:    bic x8, x8, x10
; CHECK-GI-NEXT:    orr x8, x9, x8
; CHECK-GI-NEXT:    fmov d2, x8
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i64>, <3 x i1>} @llvm.uadd.with.overflow(<3 x i64> %a, <3 x i64> %b)
  %e0 = extractvalue {<3 x i64>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i64>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i64> %e0, <3 x i64> %a
  ret <3 x i64> %r
}

define <3 x i64> @sadd_v3i64(<3 x i64> %a, <3 x i64> %b) {
; CHECK-SD-LABEL: sadd_v3i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-SD-NEXT:    // kill: def $d5 killed $d5 def $q5
; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-SD-NEXT:    mov v3.d[1], v4.d[0]
; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-SD-NEXT:    sqadd v4.2d, v2.2d, v5.2d
; CHECK-SD-NEXT:    add v5.2d, v2.2d, v5.2d
; CHECK-SD-NEXT:    sqadd v1.2d, v0.2d, v3.2d
; CHECK-SD-NEXT:    add v3.2d, v0.2d, v3.2d
; CHECK-SD-NEXT:    cmeq v1.2d, v3.2d, v1.2d
; CHECK-SD-NEXT:    bif v0.16b, v3.16b, v1.16b
; CHECK-SD-NEXT:    cmeq v1.2d, v5.2d, v4.2d
; CHECK-SD-NEXT:    bif v2.16b, v5.16b, v1.16b
; CHECK-SD-NEXT:    mov d1, v0.d[1]
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v3i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-GI-NEXT:    mov v7.16b, v0.16b
; CHECK-GI-NEXT:    mov v16.16b, v3.16b
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-GI-NEXT:    // kill: def $d5 killed $d5 def $q5
; CHECK-GI-NEXT:    mov v17.16b, v0.16b
; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
; CHECK-GI-NEXT:    fmov x10, d2
; CHECK-GI-NEXT:    fmov x11, d5
; CHECK-GI-NEXT:    movi d6, #0000000000000000
; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-GI-NEXT:    mov v7.d[1], v1.d[0]
; CHECK-GI-NEXT:    mov v16.d[1], v4.d[0]
; CHECK-GI-NEXT:    mov v17.d[1], v1.d[0]
; CHECK-GI-NEXT:    add x11, x10, x11
; CHECK-GI-NEXT:    cmlt v3.2d, v3.2d, #0
; CHECK-GI-NEXT:    cmgt v5.2d, v6.2d, v5.2d
; CHECK-GI-NEXT:    add v4.2d, v7.2d, v16.2d
; CHECK-GI-NEXT:    fmov d16, x11
; CHECK-GI-NEXT:    fmov x12, d3
; CHECK-GI-NEXT:    mov x8, v3.d[1]
; CHECK-GI-NEXT:    cmgt v7.2d, v17.2d, v4.2d
; CHECK-GI-NEXT:    cmgt v2.2d, v2.2d, v16.2d
; CHECK-GI-NEXT:    fmov s3, w12
; CHECK-GI-NEXT:    fmov x13, d7
; CHECK-GI-NEXT:    mov x9, v7.d[1]
; CHECK-GI-NEXT:    mov v3.h[1], w8
; CHECK-GI-NEXT:    fmov s7, w13
; CHECK-GI-NEXT:    mov v3.h[2], v5.h[0]
; CHECK-GI-NEXT:    mov v7.h[1], w9
; CHECK-GI-NEXT:    mov v7.h[2], v2.h[0]
; CHECK-GI-NEXT:    eor v2.8b, v3.8b, v7.8b
; CHECK-GI-NEXT:    zip1 v3.4h, v2.4h, v2.4h
; CHECK-GI-NEXT:    smov x8, v2.h[2]
; CHECK-GI-NEXT:    ushll v3.2d, v3.2s, #0
; CHECK-GI-NEXT:    and x9, x11, x8
; CHECK-GI-NEXT:    bic x8, x10, x8
; CHECK-GI-NEXT:    orr x8, x9, x8
; CHECK-GI-NEXT:    fmov d2, x8
; CHECK-GI-NEXT:    shl v3.2d, v3.2d, #63
; CHECK-GI-NEXT:    cmlt v1.2d, v3.2d, #0
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v1.16b
; CHECK-GI-NEXT:    mov d1, v0.d[1]
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i64>, <3 x i1>} @llvm.sadd.with.overflow(<3 x i64> %a, <3 x i64> %b)
  %e0 = extractvalue {<3 x i64>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i64>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i64> %e0, <3 x i64> %a
  ret <3 x i64> %r
}

define <3 x i64> @usub_v3i64(<3 x i64> %a, <3 x i64> %b) {
; CHECK-SD-LABEL: usub_v3i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-SD-NEXT:    mov v3.d[1], v4.d[0]
; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-SD-NEXT:    sub d4, d2, d5
; CHECK-SD-NEXT:    sub v1.2d, v0.2d, v3.2d
; CHECK-SD-NEXT:    cmhi v3.2d, v1.2d, v0.2d
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    cmhi v1.2d, v4.2d, v2.2d
; CHECK-SD-NEXT:    bit v2.16b, v4.16b, v1.16b
; CHECK-SD-NEXT:    mov d1, v0.d[1]
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v3i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-GI-NEXT:    mov v6.16b, v3.16b
; CHECK-GI-NEXT:    mov v7.16b, v0.16b
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-GI-NEXT:    // kill: def $d5 killed $d5 def $q5
; CHECK-GI-NEXT:    mov v16.16b, v0.16b
; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-GI-NEXT:    fmov x8, d2
; CHECK-GI-NEXT:    fmov x9, d5
; CHECK-GI-NEXT:    mov v6.d[1], v4.d[0]
; CHECK-GI-NEXT:    mov v7.d[1], v1.d[0]
; CHECK-GI-NEXT:    mov v16.d[1], v1.d[0]
; CHECK-GI-NEXT:    sub x9, x8, x9
; CHECK-GI-NEXT:    sub v1.2d, v7.2d, v6.2d
; CHECK-GI-NEXT:    cmhi v3.2d, v3.2d, v16.2d
; CHECK-GI-NEXT:    bit v0.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    cmhi v3.2d, v5.2d, v2.2d
; CHECK-GI-NEXT:    fmov x10, d3
; CHECK-GI-NEXT:    mov d1, v0.d[1]
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT:    and x9, x9, x10
; CHECK-GI-NEXT:    bic x8, x8, x10
; CHECK-GI-NEXT:    orr x8, x9, x8
; CHECK-GI-NEXT:    fmov d2, x8
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i64>, <3 x i1>} @llvm.usub.with.overflow(<3 x i64> %a, <3 x i64> %b)
  %e0 = extractvalue {<3 x i64>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i64>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i64> %e0, <3 x i64> %a
  ret <3 x i64> %r
}

define <3 x i64> @ssub_v3i64(<3 x i64> %a, <3 x i64> %b) {
; CHECK-SD-LABEL: ssub_v3i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-SD-NEXT:    // kill: def $d5 killed $d5 def $q5
; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-SD-NEXT:    mov v3.d[1], v4.d[0]
; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-SD-NEXT:    sqsub v4.2d, v2.2d, v5.2d
; CHECK-SD-NEXT:    sub v5.2d, v2.2d, v5.2d
; CHECK-SD-NEXT:    sqsub v1.2d, v0.2d, v3.2d
; CHECK-SD-NEXT:    sub v3.2d, v0.2d, v3.2d
; CHECK-SD-NEXT:    cmeq v1.2d, v3.2d, v1.2d
; CHECK-SD-NEXT:    bif v0.16b, v3.16b, v1.16b
; CHECK-SD-NEXT:    cmeq v1.2d, v5.2d, v4.2d
; CHECK-SD-NEXT:    bif v2.16b, v5.16b, v1.16b
; CHECK-SD-NEXT:    mov d1, v0.d[1]
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v3i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-GI-NEXT:    mov v7.16b, v0.16b
; CHECK-GI-NEXT:    mov v16.16b, v3.16b
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-GI-NEXT:    // kill: def $d5 killed $d5 def $q5
; CHECK-GI-NEXT:    mov v17.16b, v0.16b
; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
; CHECK-GI-NEXT:    fmov x10, d2
; CHECK-GI-NEXT:    fmov x11, d5
; CHECK-GI-NEXT:    movi d6, #0000000000000000
; CHECK-GI-NEXT:    cmgt v2.2d, v5.2d, v2.2d
; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-GI-NEXT:    mov v7.d[1], v1.d[0]
; CHECK-GI-NEXT:    mov v16.d[1], v4.d[0]
; CHECK-GI-NEXT:    mov v17.d[1], v1.d[0]
; CHECK-GI-NEXT:    sub x11, x10, x11
; CHECK-GI-NEXT:    sub v4.2d, v7.2d, v16.2d
; CHECK-GI-NEXT:    cmgt v3.2d, v3.2d, v17.2d
; CHECK-GI-NEXT:    fmov d16, x11
; CHECK-GI-NEXT:    cmlt v7.2d, v4.2d, #0
; CHECK-GI-NEXT:    fmov x12, d3
; CHECK-GI-NEXT:    mov x8, v3.d[1]
; CHECK-GI-NEXT:    cmgt v5.2d, v6.2d, v16.2d
; CHECK-GI-NEXT:    fmov x13, d7
; CHECK-GI-NEXT:    mov x9, v7.d[1]
; CHECK-GI-NEXT:    fmov s3, w12
; CHECK-GI-NEXT:    fmov s7, w13
; CHECK-GI-NEXT:    mov v3.h[1], w8
; CHECK-GI-NEXT:    mov v7.h[1], w9
; CHECK-GI-NEXT:    mov v3.h[2], v2.h[0]
; CHECK-GI-NEXT:    mov v7.h[2], v5.h[0]
; CHECK-GI-NEXT:    eor v2.8b, v3.8b, v7.8b
; CHECK-GI-NEXT:    zip1 v3.4h, v2.4h, v2.4h
; CHECK-GI-NEXT:    smov x8, v2.h[2]
; CHECK-GI-NEXT:    ushll v3.2d, v3.2s, #0
; CHECK-GI-NEXT:    and x9, x11, x8
; CHECK-GI-NEXT:    bic x8, x10, x8
; CHECK-GI-NEXT:    orr x8, x9, x8
; CHECK-GI-NEXT:    fmov d2, x8
; CHECK-GI-NEXT:    shl v3.2d, v3.2d, #63
; CHECK-GI-NEXT:    cmlt v1.2d, v3.2d, #0
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v1.16b
; CHECK-GI-NEXT:    mov d1, v0.d[1]
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i64>, <3 x i1>} @llvm.ssub.with.overflow(<3 x i64> %a, <3 x i64> %b)
  %e0 = extractvalue {<3 x i64>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i64>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i64> %e0, <3 x i64> %a
  ret <3 x i64> %r
}

define <3 x i64> @umul_v3i64(<3 x i64> %a, <3 x i64> %b) {
; CHECK-SD-LABEL: umul_v3i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-SD-NEXT:    fmov x8, d4
; CHECK-SD-NEXT:    fmov x9, d1
; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    // kill: def $d5 killed $d5 def $q5
; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-SD-NEXT:    fmov x11, d3
; CHECK-SD-NEXT:    fmov x12, d0
; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-SD-NEXT:    umulh x10, x9, x8
; CHECK-SD-NEXT:    umulh x13, x12, x11
; CHECK-SD-NEXT:    mul x11, x12, x11
; CHECK-SD-NEXT:    cmp xzr, x10
; CHECK-SD-NEXT:    fmov x10, d5
; CHECK-SD-NEXT:    mul x8, x9, x8
; CHECK-SD-NEXT:    csetm x9, ne
; CHECK-SD-NEXT:    cmp xzr, x13
; CHECK-SD-NEXT:    fmov x13, d2
; CHECK-SD-NEXT:    csetm x12, ne
; CHECK-SD-NEXT:    fmov d1, x11
; CHECK-SD-NEXT:    fmov d3, x12
; CHECK-SD-NEXT:    mov v1.d[1], x8
; CHECK-SD-NEXT:    umulh x8, x13, x10
; CHECK-SD-NEXT:    mov v3.d[1], x9
; CHECK-SD-NEXT:    mul x9, x13, x10
; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    cmp xzr, x8
; CHECK-SD-NEXT:    csetm x8, ne
; CHECK-SD-NEXT:    fmov d1, x9
; CHECK-SD-NEXT:    fmov d3, x8
; CHECK-SD-NEXT:    bit v2.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    mov d1, v0.d[1]
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v3i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-GI-NEXT:    mov v16.16b, v3.16b
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
; CHECK-GI-NEXT:    fmov x9, d3
; CHECK-GI-NEXT:    mov v7.16b, v0.16b
; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-GI-NEXT:    fmov x10, d4
; CHECK-GI-NEXT:    movi d6, #0000000000000000
; CHECK-GI-NEXT:    mov v16.d[1], v4.d[0]
; CHECK-GI-NEXT:    umulh x8, x8, x9
; CHECK-GI-NEXT:    fmov x9, d1
; CHECK-GI-NEXT:    mov v7.d[1], v1.d[0]
; CHECK-GI-NEXT:    fmov x12, d0
; CHECK-GI-NEXT:    umulh x9, x9, x10
; CHECK-GI-NEXT:    mov x10, v0.d[1]
; CHECK-GI-NEXT:    mov x11, v16.d[1]
; CHECK-GI-NEXT:    fmov x13, d16
; CHECK-GI-NEXT:    fmov d0, x8
; CHECK-GI-NEXT:    fmov x8, d2
; CHECK-GI-NEXT:    mul x12, x12, x13
; CHECK-GI-NEXT:    mov v0.d[1], x9
; CHECK-GI-NEXT:    mul x10, x10, x11
; CHECK-GI-NEXT:    fmov x11, d5
; CHECK-GI-NEXT:    umulh x9, x8, x11
; CHECK-GI-NEXT:    fmov d2, x12
; CHECK-GI-NEXT:    cmtst v0.2d, v0.2d, v0.2d
; CHECK-GI-NEXT:    mov v2.d[1], x10
; CHECK-GI-NEXT:    fmov d1, x9
; CHECK-GI-NEXT:    mul x9, x8, x11
; CHECK-GI-NEXT:    bsl v0.16b, v2.16b, v7.16b
; CHECK-GI-NEXT:    cmeq v1.2d, v1.2d, v6.2d
; CHECK-GI-NEXT:    mvn v2.16b, v1.16b
; CHECK-GI-NEXT:    mov d1, v0.d[1]
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT:    fmov x10, d2
; CHECK-GI-NEXT:    and x9, x9, x10
; CHECK-GI-NEXT:    bic x8, x8, x10
; CHECK-GI-NEXT:    orr x8, x9, x8
; CHECK-GI-NEXT:    fmov d2, x8
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i64>, <3 x i1>} @llvm.umul.with.overflow(<3 x i64> %a, <3 x i64> %b)
  %e0 = extractvalue {<3 x i64>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i64>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i64> %e0, <3 x i64> %a
  ret <3 x i64> %r
}

define <3 x i64> @smul_v3i64(<3 x i64> %a, <3 x i64> %b) {
; CHECK-SD-LABEL: smul_v3i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-SD-NEXT:    fmov x8, d4
; CHECK-SD-NEXT:    fmov x9, d1
; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT:    // kill: def $d5 killed $d5 def $q5
; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
; CHECK-SD-NEXT:    fmov x11, d0
; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-SD-NEXT:    mul x10, x9, x8
; CHECK-SD-NEXT:    smulh x8, x9, x8
; CHECK-SD-NEXT:    fmov x9, d3
; CHECK-SD-NEXT:    mul x12, x11, x9
; CHECK-SD-NEXT:    smulh x9, x11, x9
; CHECK-SD-NEXT:    cmp x8, x10, asr #63
; CHECK-SD-NEXT:    fmov x11, d5
; CHECK-SD-NEXT:    fmov x8, d2
; CHECK-SD-NEXT:    csetm x13, ne
; CHECK-SD-NEXT:    fmov d3, x12
; CHECK-SD-NEXT:    mul x14, x8, x11
; CHECK-SD-NEXT:    cmp x9, x12, asr #63
; CHECK-SD-NEXT:    csetm x9, ne
; CHECK-SD-NEXT:    smulh x8, x8, x11
; CHECK-SD-NEXT:    mov v3.d[1], x10
; CHECK-SD-NEXT:    fmov d1, x9
; CHECK-SD-NEXT:    mov v1.d[1], x13
; CHECK-SD-NEXT:    cmp x8, x14, asr #63
; CHECK-SD-NEXT:    csetm x8, ne
; CHECK-SD-NEXT:    bit v0.16b, v3.16b, v1.16b
; CHECK-SD-NEXT:    fmov d1, x8
; CHECK-SD-NEXT:    fmov d3, x14
; CHECK-SD-NEXT:    bit v2.16b, v3.16b, v1.16b
; CHECK-SD-NEXT:    mov d1, v0.d[1]
; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v3i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d3
; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
; CHECK-GI-NEXT:    fmov x12, d4
; CHECK-GI-NEXT:    mov v6.16b, v0.16b
; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
; CHECK-GI-NEXT:    umulh x10, x8, x9
; CHECK-GI-NEXT:    asr x11, x9, #63
; CHECK-GI-NEXT:    asr x14, x12, #63
; CHECK-GI-NEXT:    mov v6.d[1], v1.d[0]
; CHECK-GI-NEXT:    fmov x15, d3
; CHECK-GI-NEXT:    madd x10, x8, x11, x10
; CHECK-GI-NEXT:    fmov x11, d1
; CHECK-GI-NEXT:    asr x8, x8, #63
; CHECK-GI-NEXT:    umulh x13, x11, x12
; CHECK-GI-NEXT:    madd x8, x8, x9, x10
; CHECK-GI-NEXT:    mov x10, v0.d[1]
; CHECK-GI-NEXT:    madd x9, x11, x14, x13
; CHECK-GI-NEXT:    fmov x14, d0
; CHECK-GI-NEXT:    mov x13, v3.d[1]
; CHECK-GI-NEXT:    asr x11, x11, #63
; CHECK-GI-NEXT:    mul x14, x14, x15
; CHECK-GI-NEXT:    fmov d3, x8
; CHECK-GI-NEXT:    mul x10, x10, x13
; CHECK-GI-NEXT:    adrp x13, .LCPI101_0
; CHECK-GI-NEXT:    madd x9, x11, x12, x9
; CHECK-GI-NEXT:    fmov x11, d2
; CHECK-GI-NEXT:    fmov x12, d5
; CHECK-GI-NEXT:    fmov d0, x14
; CHECK-GI-NEXT:    ldr q2, [x13, :lo12:.LCPI101_0]
; CHECK-GI-NEXT:    umulh x13, x11, x12
; CHECK-GI-NEXT:    asr x14, x12, #63
; CHECK-GI-NEXT:    neg v2.2d, v2.2d
; CHECK-GI-NEXT:    mov v0.d[1], x10
; CHECK-GI-NEXT:    mov v3.d[1], x9
; CHECK-GI-NEXT:    asr x9, x11, #63
; CHECK-GI-NEXT:    mul x10, x11, x12
; CHECK-GI-NEXT:    madd x8, x11, x14, x13
; CHECK-GI-NEXT:    sshl v2.2d, v0.2d, v2.2d
; CHECK-GI-NEXT:    madd x8, x9, x12, x8
; CHECK-GI-NEXT:    mov w9, #191 // =0xbf
; CHECK-GI-NEXT:    cmeq v2.2d, v3.2d, v2.2d
; CHECK-GI-NEXT:    asr x9, x10, x9
; CHECK-GI-NEXT:    fmov d3, x9
; CHECK-GI-NEXT:    mvn v1.16b, v2.16b
; CHECK-GI-NEXT:    fmov d2, x8
; CHECK-GI-NEXT:    bif v0.16b, v6.16b, v1.16b
; CHECK-GI-NEXT:    cmeq v2.2d, v2.2d, v3.2d
; CHECK-GI-NEXT:    mvn v2.16b, v2.16b
; CHECK-GI-NEXT:    mov d1, v0.d[1]
; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
; CHECK-GI-NEXT:    fmov x8, d2
; CHECK-GI-NEXT:    and x9, x10, x8
; CHECK-GI-NEXT:    bic x8, x11, x8
; CHECK-GI-NEXT:    orr x8, x9, x8
; CHECK-GI-NEXT:    fmov d2, x8
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i64>, <3 x i1>} @llvm.smul.with.overflow(<3 x i64> %a, <3 x i64> %b)
  %e0 = extractvalue {<3 x i64>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i64>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i64> %e0, <3 x i64> %a
  ret <3 x i64> %r
}

define <4 x i64> @uadd_v4i64(<4 x i64> %a, <4 x i64> %b) {
; CHECK-SD-LABEL: uadd_v4i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    add v3.2d, v1.2d, v3.2d
; CHECK-SD-NEXT:    add v2.2d, v0.2d, v2.2d
; CHECK-SD-NEXT:    cmhi v4.2d, v1.2d, v3.2d
; CHECK-SD-NEXT:    cmhi v5.2d, v0.2d, v2.2d
; CHECK-SD-NEXT:    bit v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bit v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v4i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v4.2d, v0.2d, v2.2d
; CHECK-GI-NEXT:    add v5.2d, v1.2d, v3.2d
; CHECK-GI-NEXT:    cmhi v2.2d, v2.2d, v4.2d
; CHECK-GI-NEXT:    cmhi v3.2d, v3.2d, v5.2d
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v2.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v3.16b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i64>, <4 x i1>} @llvm.uadd.with.overflow(<4 x i64> %a, <4 x i64> %b)
  %e0 = extractvalue {<4 x i64>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i64>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i64> %e0, <4 x i64> %a
  ret <4 x i64> %r
}

define <4 x i64> @sadd_v4i64(<4 x i64> %a, <4 x i64> %b) {
; CHECK-SD-LABEL: sadd_v4i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqadd v4.2d, v1.2d, v3.2d
; CHECK-SD-NEXT:    sqadd v5.2d, v0.2d, v2.2d
; CHECK-SD-NEXT:    add v3.2d, v1.2d, v3.2d
; CHECK-SD-NEXT:    add v2.2d, v0.2d, v2.2d
; CHECK-SD-NEXT:    cmeq v4.2d, v3.2d, v4.2d
; CHECK-SD-NEXT:    cmeq v5.2d, v2.2d, v5.2d
; CHECK-SD-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v4i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    add v4.2d, v0.2d, v2.2d
; CHECK-GI-NEXT:    add v5.2d, v1.2d, v3.2d
; CHECK-GI-NEXT:    cmlt v2.2d, v2.2d, #0
; CHECK-GI-NEXT:    cmlt v3.2d, v3.2d, #0
; CHECK-GI-NEXT:    cmgt v6.2d, v0.2d, v4.2d
; CHECK-GI-NEXT:    cmgt v7.2d, v1.2d, v5.2d
; CHECK-GI-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
; CHECK-GI-NEXT:    eor v2.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    sshll v3.2d, v2.2s, #0
; CHECK-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v3.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i64>, <4 x i1>} @llvm.sadd.with.overflow(<4 x i64> %a, <4 x i64> %b)
  %e0 = extractvalue {<4 x i64>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i64>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i64> %e0, <4 x i64> %a
  ret <4 x i64> %r
}

define <4 x i64> @usub_v4i64(<4 x i64> %a, <4 x i64> %b) {
; CHECK-SD-LABEL: usub_v4i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub v3.2d, v1.2d, v3.2d
; CHECK-SD-NEXT:    sub v2.2d, v0.2d, v2.2d
; CHECK-SD-NEXT:    cmhi v4.2d, v3.2d, v1.2d
; CHECK-SD-NEXT:    cmhi v5.2d, v2.2d, v0.2d
; CHECK-SD-NEXT:    bit v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    bit v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v4i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v4.2d, v0.2d, v2.2d
; CHECK-GI-NEXT:    sub v5.2d, v1.2d, v3.2d
; CHECK-GI-NEXT:    cmhi v2.2d, v2.2d, v0.2d
; CHECK-GI-NEXT:    cmhi v3.2d, v3.2d, v1.2d
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v2.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v3.16b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i64>, <4 x i1>} @llvm.usub.with.overflow(<4 x i64> %a, <4 x i64> %b)
  %e0 = extractvalue {<4 x i64>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i64>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i64> %e0, <4 x i64> %a
  ret <4 x i64> %r
}

define <4 x i64> @ssub_v4i64(<4 x i64> %a, <4 x i64> %b) {
; CHECK-SD-LABEL: ssub_v4i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sqsub v4.2d, v1.2d, v3.2d
; CHECK-SD-NEXT:    sqsub v5.2d, v0.2d, v2.2d
; CHECK-SD-NEXT:    sub v3.2d, v1.2d, v3.2d
; CHECK-SD-NEXT:    sub v2.2d, v0.2d, v2.2d
; CHECK-SD-NEXT:    cmeq v4.2d, v3.2d, v4.2d
; CHECK-SD-NEXT:    cmeq v5.2d, v2.2d, v5.2d
; CHECK-SD-NEXT:    bif v0.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v4i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub v4.2d, v0.2d, v2.2d
; CHECK-GI-NEXT:    sub v5.2d, v1.2d, v3.2d
; CHECK-GI-NEXT:    cmgt v2.2d, v2.2d, v0.2d
; CHECK-GI-NEXT:    cmgt v3.2d, v3.2d, v1.2d
; CHECK-GI-NEXT:    cmlt v6.2d, v4.2d, #0
; CHECK-GI-NEXT:    cmlt v7.2d, v5.2d, #0
; CHECK-GI-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
; CHECK-GI-NEXT:    eor v2.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    sshll v3.2d, v2.2s, #0
; CHECK-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v3.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i64>, <4 x i1>} @llvm.ssub.with.overflow(<4 x i64> %a, <4 x i64> %b)
  %e0 = extractvalue {<4 x i64>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i64>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i64> %e0, <4 x i64> %a
  ret <4 x i64> %r
}

define <4 x i64> @umul_v4i64(<4 x i64> %a, <4 x i64> %b) {
; CHECK-SD-LABEL: umul_v4i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov x8, v3.d[1]
; CHECK-SD-NEXT:    mov x9, v1.d[1]
; CHECK-SD-NEXT:    fmov x10, d3
; CHECK-SD-NEXT:    fmov x11, d1
; CHECK-SD-NEXT:    mov x14, v2.d[1]
; CHECK-SD-NEXT:    mov x15, v0.d[1]
; CHECK-SD-NEXT:    fmov x16, d0
; CHECK-SD-NEXT:    mul x12, x11, x10
; CHECK-SD-NEXT:    mul x13, x9, x8
; CHECK-SD-NEXT:    umulh x8, x9, x8
; CHECK-SD-NEXT:    umulh x9, x11, x10
; CHECK-SD-NEXT:    fmov x11, d2
; CHECK-SD-NEXT:    fmov d2, x12
; CHECK-SD-NEXT:    umulh x10, x15, x14
; CHECK-SD-NEXT:    cmp xzr, x8
; CHECK-SD-NEXT:    mov v2.d[1], x13
; CHECK-SD-NEXT:    umulh x17, x16, x11
; CHECK-SD-NEXT:    csetm x8, ne
; CHECK-SD-NEXT:    cmp xzr, x9
; CHECK-SD-NEXT:    mul x11, x16, x11
; CHECK-SD-NEXT:    mul x9, x15, x14
; CHECK-SD-NEXT:    csetm x14, ne
; CHECK-SD-NEXT:    cmp xzr, x10
; CHECK-SD-NEXT:    csetm x10, ne
; CHECK-SD-NEXT:    cmp xzr, x17
; CHECK-SD-NEXT:    fmov d3, x14
; CHECK-SD-NEXT:    csetm x12, ne
; CHECK-SD-NEXT:    fmov d4, x11
; CHECK-SD-NEXT:    fmov d5, x12
; CHECK-SD-NEXT:    mov v3.d[1], x8
; CHECK-SD-NEXT:    mov v4.d[1], x9
; CHECK-SD-NEXT:    mov v5.d[1], x10
; CHECK-SD-NEXT:    bit v1.16b, v2.16b, v3.16b
; CHECK-SD-NEXT:    bit v0.16b, v4.16b, v5.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v4i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d2
; CHECK-GI-NEXT:    fmov x13, d1
; CHECK-GI-NEXT:    fmov x14, d3
; CHECK-GI-NEXT:    mov x10, v2.d[1]
; CHECK-GI-NEXT:    mov x11, v0.d[1]
; CHECK-GI-NEXT:    mov x15, v3.d[1]
; CHECK-GI-NEXT:    mov x17, v1.d[1]
; CHECK-GI-NEXT:    umulh x12, x8, x9
; CHECK-GI-NEXT:    umulh x16, x13, x14
; CHECK-GI-NEXT:    umulh x18, x11, x10
; CHECK-GI-NEXT:    fmov d2, x12
; CHECK-GI-NEXT:    umulh x0, x17, x15
; CHECK-GI-NEXT:    fmov d3, x16
; CHECK-GI-NEXT:    mul x8, x8, x9
; CHECK-GI-NEXT:    mov v2.d[1], x18
; CHECK-GI-NEXT:    mul x9, x11, x10
; CHECK-GI-NEXT:    mov v3.d[1], x0
; CHECK-GI-NEXT:    mul x10, x13, x14
; CHECK-GI-NEXT:    fmov d4, x8
; CHECK-GI-NEXT:    cmtst v2.2d, v2.2d, v2.2d
; CHECK-GI-NEXT:    mul x11, x17, x15
; CHECK-GI-NEXT:    cmtst v3.2d, v3.2d, v3.2d
; CHECK-GI-NEXT:    mov v4.d[1], x9
; CHECK-GI-NEXT:    fmov d5, x10
; CHECK-GI-NEXT:    mov v5.d[1], x11
; CHECK-GI-NEXT:    bit v0.16b, v4.16b, v2.16b
; CHECK-GI-NEXT:    bit v1.16b, v5.16b, v3.16b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i64>, <4 x i1>} @llvm.umul.with.overflow(<4 x i64> %a, <4 x i64> %b)
  %e0 = extractvalue {<4 x i64>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i64>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i64> %e0, <4 x i64> %a
  ret <4 x i64> %r
}

define <4 x i64> @smul_v4i64(<4 x i64> %a, <4 x i64> %b) {
; CHECK-SD-LABEL: smul_v4i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov x8, v3.d[1]
; CHECK-SD-NEXT:    mov x9, v1.d[1]
; CHECK-SD-NEXT:    fmov x10, d3
; CHECK-SD-NEXT:    fmov x11, d1
; CHECK-SD-NEXT:    mov x14, v2.d[1]
; CHECK-SD-NEXT:    mov x15, v0.d[1]
; CHECK-SD-NEXT:    mul x13, x9, x8
; CHECK-SD-NEXT:    smulh x8, x9, x8
; CHECK-SD-NEXT:    mul x12, x11, x10
; CHECK-SD-NEXT:    smulh x9, x11, x10
; CHECK-SD-NEXT:    cmp x8, x13, asr #63
; CHECK-SD-NEXT:    mul x10, x15, x14
; CHECK-SD-NEXT:    csetm x8, ne
; CHECK-SD-NEXT:    fmov d4, x12
; CHECK-SD-NEXT:    smulh x11, x15, x14
; CHECK-SD-NEXT:    fmov x14, d2
; CHECK-SD-NEXT:    fmov x15, d0
; CHECK-SD-NEXT:    cmp x9, x12, asr #63
; CHECK-SD-NEXT:    csetm x9, ne
; CHECK-SD-NEXT:    mov v4.d[1], x13
; CHECK-SD-NEXT:    mul x16, x15, x14
; CHECK-SD-NEXT:    fmov d2, x9
; CHECK-SD-NEXT:    smulh x14, x15, x14
; CHECK-SD-NEXT:    cmp x11, x10, asr #63
; CHECK-SD-NEXT:    csetm x9, ne
; CHECK-SD-NEXT:    mov v2.d[1], x8
; CHECK-SD-NEXT:    fmov d5, x16
; CHECK-SD-NEXT:    cmp x14, x16, asr #63
; CHECK-SD-NEXT:    bit v1.16b, v4.16b, v2.16b
; CHECK-SD-NEXT:    csetm x8, ne
; CHECK-SD-NEXT:    mov v5.d[1], x10
; CHECK-SD-NEXT:    fmov d3, x8
; CHECK-SD-NEXT:    mov v3.d[1], x9
; CHECK-SD-NEXT:    bit v0.16b, v5.16b, v3.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v4i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    mov d4, v0.d[1]
; CHECK-GI-NEXT:    mov d5, v2.d[1]
; CHECK-GI-NEXT:    fmov x8, d0
; CHECK-GI-NEXT:    fmov x9, d2
; CHECK-GI-NEXT:    mov x1, v1.d[1]
; CHECK-GI-NEXT:    asr x15, x8, #63
; CHECK-GI-NEXT:    fmov x11, d4
; CHECK-GI-NEXT:    fmov x12, d5
; CHECK-GI-NEXT:    umulh x10, x8, x9
; CHECK-GI-NEXT:    asr x13, x9, #63
; CHECK-GI-NEXT:    mov d4, v1.d[1]
; CHECK-GI-NEXT:    mov d5, v3.d[1]
; CHECK-GI-NEXT:    umulh x14, x11, x12
; CHECK-GI-NEXT:    madd x10, x8, x13, x10
; CHECK-GI-NEXT:    asr x13, x12, #63
; CHECK-GI-NEXT:    fmov x16, d5
; CHECK-GI-NEXT:    madd x13, x11, x13, x14
; CHECK-GI-NEXT:    asr x11, x11, #63
; CHECK-GI-NEXT:    asr x0, x16, #63
; CHECK-GI-NEXT:    madd x10, x15, x9, x10
; CHECK-GI-NEXT:    fmov x15, d4
; CHECK-GI-NEXT:    madd x11, x11, x12, x13
; CHECK-GI-NEXT:    fmov x12, d1
; CHECK-GI-NEXT:    fmov x13, d3
; CHECK-GI-NEXT:    umulh x18, x15, x16
; CHECK-GI-NEXT:    asr x17, x13, #63
; CHECK-GI-NEXT:    fmov d5, x10
; CHECK-GI-NEXT:    umulh x14, x12, x13
; CHECK-GI-NEXT:    madd x18, x15, x0, x18
; CHECK-GI-NEXT:    mov x0, v0.d[1]
; CHECK-GI-NEXT:    asr x15, x15, #63
; CHECK-GI-NEXT:    mov v5.d[1], x11
; CHECK-GI-NEXT:    madd x14, x12, x17, x14
; CHECK-GI-NEXT:    mov x17, v2.d[1]
; CHECK-GI-NEXT:    movi v2.2d, #0x000000000000ff
; CHECK-GI-NEXT:    mul x8, x8, x9
; CHECK-GI-NEXT:    mov x9, v3.d[1]
; CHECK-GI-NEXT:    mul x17, x0, x17
; CHECK-GI-NEXT:    neg v2.2d, v2.2d
; CHECK-GI-NEXT:    mul x0, x12, x13
; CHECK-GI-NEXT:    asr x12, x12, #63
; CHECK-GI-NEXT:    fmov d3, x8
; CHECK-GI-NEXT:    mul x9, x1, x9
; CHECK-GI-NEXT:    madd x12, x12, x13, x14
; CHECK-GI-NEXT:    mov v3.d[1], x17
; CHECK-GI-NEXT:    fmov d4, x0
; CHECK-GI-NEXT:    madd x13, x15, x16, x18
; CHECK-GI-NEXT:    mov v4.d[1], x9
; CHECK-GI-NEXT:    sshl v7.2d, v3.2d, v2.2d
; CHECK-GI-NEXT:    fmov d6, x12
; CHECK-GI-NEXT:    cmeq v5.2d, v5.2d, v7.2d
; CHECK-GI-NEXT:    mov v6.d[1], x13
; CHECK-GI-NEXT:    sshl v2.2d, v4.2d, v2.2d
; CHECK-GI-NEXT:    mvn v5.16b, v5.16b
; CHECK-GI-NEXT:    cmeq v2.2d, v6.2d, v2.2d
; CHECK-GI-NEXT:    bit v0.16b, v3.16b, v5.16b
; CHECK-GI-NEXT:    mvn v2.16b, v2.16b
; CHECK-GI-NEXT:    bit v1.16b, v4.16b, v2.16b
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i64>, <4 x i1>} @llvm.smul.with.overflow(<4 x i64> %a, <4 x i64> %b)
  %e0 = extractvalue {<4 x i64>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i64>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i64> %e0, <4 x i64> %a
  ret <4 x i64> %r
}

define <2 x i128> @uadd_v2i128(<2 x i128> %a, <2 x i128> %b) {
; CHECK-SD-LABEL: uadd_v2i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    adds x8, x0, x4
; CHECK-SD-NEXT:    adcs x9, x1, x5
; CHECK-SD-NEXT:    csel x1, x9, x1, hs
; CHECK-SD-NEXT:    csel x0, x8, x0, hs
; CHECK-SD-NEXT:    adds x8, x2, x6
; CHECK-SD-NEXT:    adcs x9, x3, x7
; CHECK-SD-NEXT:    csel x2, x8, x2, hs
; CHECK-SD-NEXT:    csel x3, x9, x3, hs
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v2i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    adds x8, x0, x4
; CHECK-GI-NEXT:    adc x9, x1, x5
; CHECK-GI-NEXT:    adds x10, x2, x6
; CHECK-GI-NEXT:    adc x11, x3, x7
; CHECK-GI-NEXT:    cmp x8, x4
; CHECK-GI-NEXT:    cset w12, lo
; CHECK-GI-NEXT:    cmp x9, x5
; CHECK-GI-NEXT:    cset w13, lo
; CHECK-GI-NEXT:    csel w12, w12, w13, eq
; CHECK-GI-NEXT:    cmp x10, x6
; CHECK-GI-NEXT:    cset w13, lo
; CHECK-GI-NEXT:    cmp x11, x7
; CHECK-GI-NEXT:    cset w14, lo
; CHECK-GI-NEXT:    csel w13, w13, w14, eq
; CHECK-GI-NEXT:    tst w12, #0x1
; CHECK-GI-NEXT:    csel x0, x8, x0, ne
; CHECK-GI-NEXT:    csel x1, x9, x1, ne
; CHECK-GI-NEXT:    tst w13, #0x1
; CHECK-GI-NEXT:    csel x2, x10, x2, ne
; CHECK-GI-NEXT:    csel x3, x11, x3, ne
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i128>, <2 x i1>} @llvm.uadd.with.overflow(<2 x i128> %a, <2 x i128> %b)
  %e0 = extractvalue {<2 x i128>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i128>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i128> %e0, <2 x i128> %a
  ret <2 x i128> %r
}

define <2 x i128> @sadd_v2i128(<2 x i128> %a, <2 x i128> %b) {
; CHECK-SD-LABEL: sadd_v2i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    adds x8, x0, x4
; CHECK-SD-NEXT:    adcs x9, x1, x5
; CHECK-SD-NEXT:    csel x1, x9, x1, vs
; CHECK-SD-NEXT:    csel x0, x8, x0, vs
; CHECK-SD-NEXT:    adds x8, x2, x6
; CHECK-SD-NEXT:    adcs x9, x3, x7
; CHECK-SD-NEXT:    csel x2, x8, x2, vs
; CHECK-SD-NEXT:    csel x3, x9, x3, vs
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v2i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    adds x8, x0, x4
; CHECK-GI-NEXT:    adc x9, x1, x5
; CHECK-GI-NEXT:    adds x10, x2, x6
; CHECK-GI-NEXT:    adc x11, x3, x7
; CHECK-GI-NEXT:    cmp x8, x0
; CHECK-GI-NEXT:    cset w12, lo
; CHECK-GI-NEXT:    cmp x9, x1
; CHECK-GI-NEXT:    cset w13, lt
; CHECK-GI-NEXT:    csel w12, w12, w13, eq
; CHECK-GI-NEXT:    cmp x10, x2
; CHECK-GI-NEXT:    cset w13, lo
; CHECK-GI-NEXT:    cmp x11, x3
; CHECK-GI-NEXT:    fmov s1, w12
; CHECK-GI-NEXT:    cset w14, lt
; CHECK-GI-NEXT:    csel w13, w13, w14, eq
; CHECK-GI-NEXT:    cmp x5, #0
; CHECK-GI-NEXT:    cset w14, mi
; CHECK-GI-NEXT:    mov v1.s[1], w13
; CHECK-GI-NEXT:    csel w14, wzr, w14, eq
; CHECK-GI-NEXT:    cmp x7, #0
; CHECK-GI-NEXT:    fmov s0, w14
; CHECK-GI-NEXT:    cset w15, mi
; CHECK-GI-NEXT:    csel w15, wzr, w15, eq
; CHECK-GI-NEXT:    mov v0.s[1], w15
; CHECK-GI-NEXT:    eor v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    mov s1, v0.s[1]
; CHECK-GI-NEXT:    fmov w12, s0
; CHECK-GI-NEXT:    tst w12, #0x1
; CHECK-GI-NEXT:    csel x0, x8, x0, ne
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    tst w12, #0x1
; CHECK-GI-NEXT:    csel x1, x9, x1, ne
; CHECK-GI-NEXT:    tst w8, #0x1
; CHECK-GI-NEXT:    csel x2, x10, x2, ne
; CHECK-GI-NEXT:    tst w8, #0x1
; CHECK-GI-NEXT:    csel x3, x11, x3, ne
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i128>, <2 x i1>} @llvm.sadd.with.overflow(<2 x i128> %a, <2 x i128> %b)
  %e0 = extractvalue {<2 x i128>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i128>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i128> %e0, <2 x i128> %a
  ret <2 x i128> %r
}

define <2 x i128> @usub_v2i128(<2 x i128> %a, <2 x i128> %b) {
; CHECK-SD-LABEL: usub_v2i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    subs x8, x0, x4
; CHECK-SD-NEXT:    sbcs x9, x1, x5
; CHECK-SD-NEXT:    csel x1, x9, x1, lo
; CHECK-SD-NEXT:    csel x0, x8, x0, lo
; CHECK-SD-NEXT:    subs x8, x2, x6
; CHECK-SD-NEXT:    sbcs x9, x3, x7
; CHECK-SD-NEXT:    csel x2, x8, x2, lo
; CHECK-SD-NEXT:    csel x3, x9, x3, lo
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v2i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    subs x8, x0, x4
; CHECK-GI-NEXT:    sbc x9, x1, x5
; CHECK-GI-NEXT:    subs x10, x2, x6
; CHECK-GI-NEXT:    sbc x11, x3, x7
; CHECK-GI-NEXT:    cmp x0, x4
; CHECK-GI-NEXT:    cset w12, lo
; CHECK-GI-NEXT:    cmp x1, x5
; CHECK-GI-NEXT:    cset w13, lo
; CHECK-GI-NEXT:    csel w12, w12, w13, eq
; CHECK-GI-NEXT:    cmp x2, x6
; CHECK-GI-NEXT:    cset w13, lo
; CHECK-GI-NEXT:    cmp x3, x7
; CHECK-GI-NEXT:    cset w14, lo
; CHECK-GI-NEXT:    csel w13, w13, w14, eq
; CHECK-GI-NEXT:    tst w12, #0x1
; CHECK-GI-NEXT:    csel x0, x8, x0, ne
; CHECK-GI-NEXT:    csel x1, x9, x1, ne
; CHECK-GI-NEXT:    tst w13, #0x1
; CHECK-GI-NEXT:    csel x2, x10, x2, ne
; CHECK-GI-NEXT:    csel x3, x11, x3, ne
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i128>, <2 x i1>} @llvm.usub.with.overflow(<2 x i128> %a, <2 x i128> %b)
  %e0 = extractvalue {<2 x i128>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i128>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i128> %e0, <2 x i128> %a
  ret <2 x i128> %r
}

define <2 x i128> @ssub_v2i128(<2 x i128> %a, <2 x i128> %b) {
; CHECK-SD-LABEL: ssub_v2i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    subs x8, x0, x4
; CHECK-SD-NEXT:    sbcs x9, x1, x5
; CHECK-SD-NEXT:    csel x1, x9, x1, vs
; CHECK-SD-NEXT:    csel x0, x8, x0, vs
; CHECK-SD-NEXT:    subs x8, x2, x6
; CHECK-SD-NEXT:    sbcs x9, x3, x7
; CHECK-SD-NEXT:    csel x2, x8, x2, vs
; CHECK-SD-NEXT:    csel x3, x9, x3, vs
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v2i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    subs x8, x0, x4
; CHECK-GI-NEXT:    sbc x9, x1, x5
; CHECK-GI-NEXT:    subs x10, x2, x6
; CHECK-GI-NEXT:    sbc x11, x3, x7
; CHECK-GI-NEXT:    cmp x0, x4
; CHECK-GI-NEXT:    cset w12, lo
; CHECK-GI-NEXT:    cmp x1, x5
; CHECK-GI-NEXT:    cset w13, lt
; CHECK-GI-NEXT:    csel w12, w12, w13, eq
; CHECK-GI-NEXT:    cmp x2, x6
; CHECK-GI-NEXT:    cset w13, lo
; CHECK-GI-NEXT:    cmp x3, x7
; CHECK-GI-NEXT:    fmov s0, w12
; CHECK-GI-NEXT:    cset w14, lt
; CHECK-GI-NEXT:    csel w13, w13, w14, eq
; CHECK-GI-NEXT:    cmp x9, #0
; CHECK-GI-NEXT:    cset w14, mi
; CHECK-GI-NEXT:    mov v0.s[1], w13
; CHECK-GI-NEXT:    csel w14, wzr, w14, eq
; CHECK-GI-NEXT:    cmp x11, #0
; CHECK-GI-NEXT:    fmov s1, w14
; CHECK-GI-NEXT:    cset w15, mi
; CHECK-GI-NEXT:    csel w15, wzr, w15, eq
; CHECK-GI-NEXT:    mov v1.s[1], w15
; CHECK-GI-NEXT:    eor v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    mov s1, v0.s[1]
; CHECK-GI-NEXT:    fmov w12, s0
; CHECK-GI-NEXT:    tst w12, #0x1
; CHECK-GI-NEXT:    csel x0, x8, x0, ne
; CHECK-GI-NEXT:    fmov w8, s1
; CHECK-GI-NEXT:    tst w12, #0x1
; CHECK-GI-NEXT:    csel x1, x9, x1, ne
; CHECK-GI-NEXT:    tst w8, #0x1
; CHECK-GI-NEXT:    csel x2, x10, x2, ne
; CHECK-GI-NEXT:    tst w8, #0x1
; CHECK-GI-NEXT:    csel x3, x11, x3, ne
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i128>, <2 x i1>} @llvm.ssub.with.overflow(<2 x i128> %a, <2 x i128> %b)
  %e0 = extractvalue {<2 x i128>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i128>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i128> %e0, <2 x i128> %a
  ret <2 x i128> %r
}

define <2 x i128> @umul_v2i128(<2 x i128> %a, <2 x i128> %b) {
; CHECK-SD-LABEL: umul_v2i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mul x9, x7, x2
; CHECK-SD-NEXT:    cmp x3, #0
; CHECK-SD-NEXT:    ccmp x7, #0, #4, ne
; CHECK-SD-NEXT:    umulh x10, x3, x6
; CHECK-SD-NEXT:    umulh x8, x7, x2
; CHECK-SD-NEXT:    madd x9, x3, x6, x9
; CHECK-SD-NEXT:    ccmp xzr, x10, #0, eq
; CHECK-SD-NEXT:    umulh x11, x2, x6
; CHECK-SD-NEXT:    ccmp xzr, x8, #0, eq
; CHECK-SD-NEXT:    mul x13, x5, x0
; CHECK-SD-NEXT:    cset w8, ne
; CHECK-SD-NEXT:    umulh x14, x1, x4
; CHECK-SD-NEXT:    adds x9, x11, x9
; CHECK-SD-NEXT:    umulh x12, x5, x0
; CHECK-SD-NEXT:    csinc w8, w8, wzr, lo
; CHECK-SD-NEXT:    cmp x1, #0
; CHECK-SD-NEXT:    ccmp x5, #0, #4, ne
; CHECK-SD-NEXT:    madd x10, x1, x4, x13
; CHECK-SD-NEXT:    ccmp xzr, x14, #0, eq
; CHECK-SD-NEXT:    umulh x11, x0, x4
; CHECK-SD-NEXT:    ccmp xzr, x12, #0, eq
; CHECK-SD-NEXT:    mul x13, x0, x4
; CHECK-SD-NEXT:    cset w14, ne
; CHECK-SD-NEXT:    mul x12, x2, x6
; CHECK-SD-NEXT:    adds x10, x11, x10
; CHECK-SD-NEXT:    csinc w11, w14, wzr, lo
; CHECK-SD-NEXT:    cmp w11, #0
; CHECK-SD-NEXT:    csel x0, x13, x0, ne
; CHECK-SD-NEXT:    csel x1, x10, x1, ne
; CHECK-SD-NEXT:    cmp w8, #0
; CHECK-SD-NEXT:    csel x2, x12, x2, ne
; CHECK-SD-NEXT:    csel x3, x9, x3, ne
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v2i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    stp x22, x21, [sp, #-32]! // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    umulh x10, x0, x4
; CHECK-GI-NEXT:    mul x16, x0, x5
; CHECK-GI-NEXT:    mul x11, x1, x4
; CHECK-GI-NEXT:    umulh x13, x1, x4
; CHECK-GI-NEXT:    adds x16, x10, x16
; CHECK-GI-NEXT:    umulh x15, x0, x5
; CHECK-GI-NEXT:    mul x14, x1, x5
; CHECK-GI-NEXT:    umulh x12, x1, x5
; CHECK-GI-NEXT:    umulh x17, x2, x6
; CHECK-GI-NEXT:    mul x18, x2, x7
; CHECK-GI-NEXT:    madd x10, x0, x5, x10
; CHECK-GI-NEXT:    cset w5, hs
; CHECK-GI-NEXT:    cmn x16, x11
; CHECK-GI-NEXT:    cset w11, hs
; CHECK-GI-NEXT:    adds x13, x13, x15
; CHECK-GI-NEXT:    and x5, x5, #0x1
; CHECK-GI-NEXT:    mul x19, x3, x6
; CHECK-GI-NEXT:    and x11, x11, #0x1
; CHECK-GI-NEXT:    cset w15, hs
; CHECK-GI-NEXT:    adds x13, x13, x14
; CHECK-GI-NEXT:    add x11, x5, x11
; CHECK-GI-NEXT:    and x15, x15, #0x1
; CHECK-GI-NEXT:    umulh x20, x3, x6
; CHECK-GI-NEXT:    cset w5, hs
; CHECK-GI-NEXT:    adds x11, x13, x11
; CHECK-GI-NEXT:    and x5, x5, #0x1
; CHECK-GI-NEXT:    umulh x21, x2, x7
; CHECK-GI-NEXT:    add x15, x15, x5
; CHECK-GI-NEXT:    cset w5, hs
; CHECK-GI-NEXT:    adds x18, x17, x18
; CHECK-GI-NEXT:    and x5, x5, #0x1
; CHECK-GI-NEXT:    umulh x9, x4, xzr
; CHECK-GI-NEXT:    add x15, x15, x5
; CHECK-GI-NEXT:    mul x22, x3, x7
; CHECK-GI-NEXT:    umulh x8, x0, xzr
; CHECK-GI-NEXT:    add x9, x9, x12
; CHECK-GI-NEXT:    umulh x14, x3, x7
; CHECK-GI-NEXT:    madd x17, x2, x7, x17
; CHECK-GI-NEXT:    cset w7, hs
; CHECK-GI-NEXT:    cmn x18, x19
; CHECK-GI-NEXT:    cset w18, hs
; CHECK-GI-NEXT:    adds x5, x20, x21
; CHECK-GI-NEXT:    and x7, x7, #0x1
; CHECK-GI-NEXT:    umulh x16, x6, xzr
; CHECK-GI-NEXT:    cset w12, hs
; CHECK-GI-NEXT:    adds x5, x5, x22
; CHECK-GI-NEXT:    and x18, x18, #0x1
; CHECK-GI-NEXT:    and x12, x12, #0x1
; CHECK-GI-NEXT:    add x8, x9, x8
; CHECK-GI-NEXT:    umulh x13, x2, xzr
; CHECK-GI-NEXT:    add x18, x7, x18
; CHECK-GI-NEXT:    cset w7, hs
; CHECK-GI-NEXT:    and x7, x7, #0x1
; CHECK-GI-NEXT:    adds x18, x5, x18
; CHECK-GI-NEXT:    add x8, x8, x15
; CHECK-GI-NEXT:    madd x10, x1, x4, x10
; CHECK-GI-NEXT:    add x9, x12, x7
; CHECK-GI-NEXT:    cset w12, hs
; CHECK-GI-NEXT:    add x14, x16, x14
; CHECK-GI-NEXT:    and x12, x12, #0x1
; CHECK-GI-NEXT:    orr x8, x11, x8
; CHECK-GI-NEXT:    mul x4, x0, x4
; CHECK-GI-NEXT:    add x9, x9, x12
; CHECK-GI-NEXT:    cmp x8, #0
; CHECK-GI-NEXT:    add x12, x14, x13
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mul x5, x2, x6
; CHECK-GI-NEXT:    add x9, x12, x9
; CHECK-GI-NEXT:    orr x9, x18, x9
; CHECK-GI-NEXT:    csel x1, x10, x1, ne
; CHECK-GI-NEXT:    madd x16, x3, x6, x17
; CHECK-GI-NEXT:    csel x0, x4, x0, ne
; CHECK-GI-NEXT:    cmp x9, #0
; CHECK-GI-NEXT:    csel x2, x5, x2, ne
; CHECK-GI-NEXT:    csel x3, x16, x3, ne
; CHECK-GI-NEXT:    ldp x22, x21, [sp], #32 // 16-byte Folded Reload
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow(<2 x i128> %a, <2 x i128> %b)
  %e0 = extractvalue {<2 x i128>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i128>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i128> %e0, <2 x i128> %a
  ret <2 x i128> %r
}

define <2 x i128> @smul_v2i128(<2 x i128> %a, <2 x i128> %b) {
; CHECK-SD-LABEL: smul_v2i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    stp x20, x19, [sp, #-16]! // 16-byte Folded Spill
; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
; CHECK-SD-NEXT:    .cfi_offset w19, -8
; CHECK-SD-NEXT:    .cfi_offset w20, -16
; CHECK-SD-NEXT:    asr x9, x3, #63
; CHECK-SD-NEXT:    umulh x10, x2, x6
; CHECK-SD-NEXT:    asr x13, x7, #63
; CHECK-SD-NEXT:    asr x18, x1, #63
; CHECK-SD-NEXT:    mul x11, x3, x6
; CHECK-SD-NEXT:    umulh x8, x3, x6
; CHECK-SD-NEXT:    mul x9, x9, x6
; CHECK-SD-NEXT:    adds x10, x11, x10
; CHECK-SD-NEXT:    mul x14, x2, x7
; CHECK-SD-NEXT:    umulh x12, x2, x7
; CHECK-SD-NEXT:    adc x8, x8, x9
; CHECK-SD-NEXT:    mul x13, x2, x13
; CHECK-SD-NEXT:    adds x9, x14, x10
; CHECK-SD-NEXT:    asr x10, x5, #63
; CHECK-SD-NEXT:    mul x16, x3, x7
; CHECK-SD-NEXT:    smulh x15, x3, x7
; CHECK-SD-NEXT:    adc x12, x12, x13
; CHECK-SD-NEXT:    asr x13, x8, #63
; CHECK-SD-NEXT:    umulh x7, x0, x4
; CHECK-SD-NEXT:    asr x20, x12, #63
; CHECK-SD-NEXT:    adds x8, x8, x12
; CHECK-SD-NEXT:    mul x19, x1, x4
; CHECK-SD-NEXT:    adc x12, x13, x20
; CHECK-SD-NEXT:    adds x8, x16, x8
; CHECK-SD-NEXT:    adc x12, x15, x12
; CHECK-SD-NEXT:    umulh x17, x1, x4
; CHECK-SD-NEXT:    mul x18, x18, x4
; CHECK-SD-NEXT:    adds x15, x19, x7
; CHECK-SD-NEXT:    mul x14, x0, x5
; CHECK-SD-NEXT:    umulh x11, x0, x5
; CHECK-SD-NEXT:    adc x16, x17, x18
; CHECK-SD-NEXT:    mul x10, x0, x10
; CHECK-SD-NEXT:    adds x14, x14, x15
; CHECK-SD-NEXT:    mul x13, x1, x5
; CHECK-SD-NEXT:    smulh x15, x1, x5
; CHECK-SD-NEXT:    adc x10, x11, x10
; CHECK-SD-NEXT:    asr x11, x16, #63
; CHECK-SD-NEXT:    asr x17, x10, #63
; CHECK-SD-NEXT:    adds x10, x16, x10
; CHECK-SD-NEXT:    mul x16, x0, x4
; CHECK-SD-NEXT:    adc x11, x11, x17
; CHECK-SD-NEXT:    adds x10, x13, x10
; CHECK-SD-NEXT:    asr x13, x14, #63
; CHECK-SD-NEXT:    asr x17, x9, #63
; CHECK-SD-NEXT:    adc x11, x15, x11
; CHECK-SD-NEXT:    mul x15, x2, x6
; CHECK-SD-NEXT:    eor x11, x11, x13
; CHECK-SD-NEXT:    eor x10, x10, x13
; CHECK-SD-NEXT:    eor x12, x12, x17
; CHECK-SD-NEXT:    eor x8, x8, x17
; CHECK-SD-NEXT:    orr x10, x10, x11
; CHECK-SD-NEXT:    orr x8, x8, x12
; CHECK-SD-NEXT:    cmp x10, #0
; CHECK-SD-NEXT:    csel x0, x16, x0, ne
; CHECK-SD-NEXT:    csel x1, x14, x1, ne
; CHECK-SD-NEXT:    cmp x8, #0
; CHECK-SD-NEXT:    csel x2, x15, x2, ne
; CHECK-SD-NEXT:    csel x3, x9, x3, ne
; CHECK-SD-NEXT:    ldp x20, x19, [sp], #16 // 16-byte Folded Reload
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v2i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    stp x26, x25, [sp, #-64]! // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x24, x23, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    .cfi_offset w23, -40
; CHECK-GI-NEXT:    .cfi_offset w24, -48
; CHECK-GI-NEXT:    .cfi_offset w25, -56
; CHECK-GI-NEXT:    .cfi_offset w26, -64
; CHECK-GI-NEXT:    asr x9, x1, #63
; CHECK-GI-NEXT:    asr x13, x5, #63
; CHECK-GI-NEXT:    umulh x14, x1, x5
; CHECK-GI-NEXT:    asr x22, x7, #63
; CHECK-GI-NEXT:    umulh x12, x9, x4
; CHECK-GI-NEXT:    umulh x15, x0, x13
; CHECK-GI-NEXT:    mul x20, x0, x13
; CHECK-GI-NEXT:    add x12, x12, x14
; CHECK-GI-NEXT:    umulh x23, x3, x7
; CHECK-GI-NEXT:    add x12, x12, x15
; CHECK-GI-NEXT:    madd x12, x0, x13, x12
; CHECK-GI-NEXT:    umulh x24, x2, x22
; CHECK-GI-NEXT:    madd x12, x1, x13, x12
; CHECK-GI-NEXT:    asr x13, x3, #63
; CHECK-GI-NEXT:    umulh x21, x13, x6
; CHECK-GI-NEXT:    umulh x8, x0, x4
; CHECK-GI-NEXT:    mul x10, x0, x5
; CHECK-GI-NEXT:    add x21, x21, x23
; CHECK-GI-NEXT:    mul x11, x1, x4
; CHECK-GI-NEXT:    add x21, x21, x24
; CHECK-GI-NEXT:    umulh x18, x1, x4
; CHECK-GI-NEXT:    adds x10, x8, x10
; CHECK-GI-NEXT:    umulh x19, x0, x5
; CHECK-GI-NEXT:    cset w23, hs
; CHECK-GI-NEXT:    cmn x10, x11
; CHECK-GI-NEXT:    mul x16, x1, x5
; CHECK-GI-NEXT:    cset w11, hs
; CHECK-GI-NEXT:    and x11, x11, #0x1
; CHECK-GI-NEXT:    madd x12, x9, x5, x12
; CHECK-GI-NEXT:    adds x18, x18, x19
; CHECK-GI-NEXT:    mul x17, x9, x4
; CHECK-GI-NEXT:    cset w19, hs
; CHECK-GI-NEXT:    adds x18, x18, x20
; CHECK-GI-NEXT:    and x19, x19, #0x1
; CHECK-GI-NEXT:    madd x21, x2, x22, x21
; CHECK-GI-NEXT:    madd x9, x9, x4, x12
; CHECK-GI-NEXT:    umulh x12, x2, x6
; CHECK-GI-NEXT:    mul x15, x2, x7
; CHECK-GI-NEXT:    mul x14, x3, x6
; CHECK-GI-NEXT:    madd x20, x3, x22, x21
; CHECK-GI-NEXT:    and x21, x23, #0x1
; CHECK-GI-NEXT:    add x11, x21, x11
; CHECK-GI-NEXT:    umulh x25, x3, x6
; CHECK-GI-NEXT:    umulh x26, x2, x7
; CHECK-GI-NEXT:    mul x10, x2, x22
; CHECK-GI-NEXT:    cset w22, hs
; CHECK-GI-NEXT:    adds x16, x18, x16
; CHECK-GI-NEXT:    cset w18, hs
; CHECK-GI-NEXT:    adds x16, x16, x17
; CHECK-GI-NEXT:    and x22, x22, #0x1
; CHECK-GI-NEXT:    madd x17, x13, x7, x20
; CHECK-GI-NEXT:    cset w20, hs
; CHECK-GI-NEXT:    adds x11, x16, x11
; CHECK-GI-NEXT:    cset w16, hs
; CHECK-GI-NEXT:    adds x15, x12, x15
; CHECK-GI-NEXT:    add x19, x19, x22
; CHECK-GI-NEXT:    mul x23, x3, x7
; CHECK-GI-NEXT:    and x18, x18, #0x1
; CHECK-GI-NEXT:    cset w22, hs
; CHECK-GI-NEXT:    cmn x15, x14
; CHECK-GI-NEXT:    add x14, x19, x18
; CHECK-GI-NEXT:    and x16, x16, #0x1
; CHECK-GI-NEXT:    mul x21, x13, x6
; CHECK-GI-NEXT:    cset w15, hs
; CHECK-GI-NEXT:    adds x18, x25, x26
; CHECK-GI-NEXT:    and x15, x15, #0x1
; CHECK-GI-NEXT:    madd x13, x13, x6, x17
; CHECK-GI-NEXT:    cset w17, hs
; CHECK-GI-NEXT:    adds x10, x18, x10
; CHECK-GI-NEXT:    and x18, x20, #0x1
; CHECK-GI-NEXT:    and x17, x17, #0x1
; CHECK-GI-NEXT:    add x14, x14, x18
; CHECK-GI-NEXT:    madd x8, x0, x5, x8
; CHECK-GI-NEXT:    cset w18, hs
; CHECK-GI-NEXT:    adds x10, x10, x23
; CHECK-GI-NEXT:    and x18, x18, #0x1
; CHECK-GI-NEXT:    and x5, x22, #0x1
; CHECK-GI-NEXT:    madd x12, x2, x7, x12
; CHECK-GI-NEXT:    add x17, x17, x18
; CHECK-GI-NEXT:    cset w18, hs
; CHECK-GI-NEXT:    add x15, x5, x15
; CHECK-GI-NEXT:    and x18, x18, #0x1
; CHECK-GI-NEXT:    adds x10, x10, x21
; CHECK-GI-NEXT:    madd x8, x1, x4, x8
; CHECK-GI-NEXT:    add x17, x17, x18
; CHECK-GI-NEXT:    cset w18, hs
; CHECK-GI-NEXT:    adds x10, x10, x15
; CHECK-GI-NEXT:    and x15, x18, #0x1
; CHECK-GI-NEXT:    add x14, x14, x16
; CHECK-GI-NEXT:    madd x12, x3, x6, x12
; CHECK-GI-NEXT:    cset w18, hs
; CHECK-GI-NEXT:    add x15, x17, x15
; CHECK-GI-NEXT:    and x17, x18, #0x1
; CHECK-GI-NEXT:    add x9, x9, x14
; CHECK-GI-NEXT:    add x15, x15, x17
; CHECK-GI-NEXT:    mul x16, x0, x4
; CHECK-GI-NEXT:    add x13, x13, x15
; CHECK-GI-NEXT:    eor x11, x11, x8, asr #63
; CHECK-GI-NEXT:    eor x9, x9, x8, asr #63
; CHECK-GI-NEXT:    mul x14, x2, x6
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT:    eor x10, x10, x12, asr #63
; CHECK-GI-NEXT:    eor x13, x13, x12, asr #63
; CHECK-GI-NEXT:    orr x9, x11, x9
; CHECK-GI-NEXT:    cmp x9, #0
; CHECK-GI-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    orr x10, x10, x13
; CHECK-GI-NEXT:    ldp x24, x23, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x0, x16, x0, ne
; CHECK-GI-NEXT:    csel x1, x8, x1, ne
; CHECK-GI-NEXT:    cmp x10, #0
; CHECK-GI-NEXT:    csel x2, x14, x2, ne
; CHECK-GI-NEXT:    csel x3, x12, x3, ne
; CHECK-GI-NEXT:    ldp x26, x25, [sp], #64 // 16-byte Folded Reload
; CHECK-GI-NEXT:    ret
  %o = call {<2 x i128>, <2 x i1>} @llvm.smul.with.overflow(<2 x i128> %a, <2 x i128> %b)
  %e0 = extractvalue {<2 x i128>, <2 x i1>} %o, 0
  %e1 = extractvalue {<2 x i128>, <2 x i1>} %o, 1
  %r = select <2 x i1> %e1, <2 x i128> %e0, <2 x i128> %a
  ret <2 x i128> %r
}

define <3 x i128> @uadd_v3i128(<3 x i128> %a, <3 x i128> %b) {
; CHECK-SD-LABEL: uadd_v3i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    ldp x8, x9, [sp]
; CHECK-SD-NEXT:    adds x8, x2, x8
; CHECK-SD-NEXT:    adcs x9, x3, x9
; CHECK-SD-NEXT:    csel x3, x9, x3, hs
; CHECK-SD-NEXT:    ldp x9, x10, [sp, #16]
; CHECK-SD-NEXT:    csel x2, x8, x2, hs
; CHECK-SD-NEXT:    adds x8, x0, x6
; CHECK-SD-NEXT:    adcs x11, x1, x7
; CHECK-SD-NEXT:    csel x1, x11, x1, hs
; CHECK-SD-NEXT:    csel x0, x8, x0, hs
; CHECK-SD-NEXT:    adds x8, x4, x9
; CHECK-SD-NEXT:    adcs x9, x5, x10
; CHECK-SD-NEXT:    csel x4, x8, x4, hs
; CHECK-SD-NEXT:    csel x5, x9, x5, hs
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v3i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    ldp x8, x9, [sp]
; CHECK-GI-NEXT:    adds x10, x0, x6
; CHECK-GI-NEXT:    ldp x11, x12, [sp, #16]
; CHECK-GI-NEXT:    adc x13, x1, x7
; CHECK-GI-NEXT:    adds x14, x2, x8
; CHECK-GI-NEXT:    adc x15, x3, x9
; CHECK-GI-NEXT:    adds x16, x4, x11
; CHECK-GI-NEXT:    adc x17, x5, x12
; CHECK-GI-NEXT:    cmp x10, x6
; CHECK-GI-NEXT:    cset w18, lo
; CHECK-GI-NEXT:    cmp x13, x7
; CHECK-GI-NEXT:    cset w6, lo
; CHECK-GI-NEXT:    csel w18, w18, w6, eq
; CHECK-GI-NEXT:    cmp x14, x8
; CHECK-GI-NEXT:    cset w8, lo
; CHECK-GI-NEXT:    cmp x15, x9
; CHECK-GI-NEXT:    cset w9, lo
; CHECK-GI-NEXT:    csel w8, w8, w9, eq
; CHECK-GI-NEXT:    cmp x16, x11
; CHECK-GI-NEXT:    cset w9, lo
; CHECK-GI-NEXT:    cmp x17, x12
; CHECK-GI-NEXT:    cset w11, lo
; CHECK-GI-NEXT:    csel w9, w9, w11, eq
; CHECK-GI-NEXT:    tst w18, #0x1
; CHECK-GI-NEXT:    csel x0, x10, x0, ne
; CHECK-GI-NEXT:    csel x1, x13, x1, ne
; CHECK-GI-NEXT:    tst w8, #0x1
; CHECK-GI-NEXT:    csel x2, x14, x2, ne
; CHECK-GI-NEXT:    csel x3, x15, x3, ne
; CHECK-GI-NEXT:    tst w9, #0x1
; CHECK-GI-NEXT:    csel x4, x16, x4, ne
; CHECK-GI-NEXT:    csel x5, x17, x5, ne
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i128>, <3 x i1>} @llvm.uadd.with.overflow(<3 x i128> %a, <3 x i128> %b)
  %e0 = extractvalue {<3 x i128>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i128>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i128> %e0, <3 x i128> %a
  ret <3 x i128> %r
}

define <3 x i128> @sadd_v3i128(<3 x i128> %a, <3 x i128> %b) {
; CHECK-SD-LABEL: sadd_v3i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    ldp x8, x9, [sp]
; CHECK-SD-NEXT:    adds x8, x2, x8
; CHECK-SD-NEXT:    adcs x9, x3, x9
; CHECK-SD-NEXT:    csel x3, x9, x3, vs
; CHECK-SD-NEXT:    ldp x9, x10, [sp, #16]
; CHECK-SD-NEXT:    csel x2, x8, x2, vs
; CHECK-SD-NEXT:    adds x8, x0, x6
; CHECK-SD-NEXT:    adcs x11, x1, x7
; CHECK-SD-NEXT:    csel x1, x11, x1, vs
; CHECK-SD-NEXT:    csel x0, x8, x0, vs
; CHECK-SD-NEXT:    adds x8, x4, x9
; CHECK-SD-NEXT:    adcs x9, x5, x10
; CHECK-SD-NEXT:    csel x4, x8, x4, vs
; CHECK-SD-NEXT:    csel x5, x9, x5, vs
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v3i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    ldp x8, x14, [sp]
; CHECK-GI-NEXT:    adds x10, x0, x6
; CHECK-GI-NEXT:    ldp x9, x15, [sp, #16]
; CHECK-GI-NEXT:    adc x13, x1, x7
; CHECK-GI-NEXT:    adds x11, x2, x8
; CHECK-GI-NEXT:    adc x12, x3, x14
; CHECK-GI-NEXT:    adds x8, x4, x9
; CHECK-GI-NEXT:    adc x9, x5, x15
; CHECK-GI-NEXT:    cmp x10, x0
; CHECK-GI-NEXT:    cset w16, lo
; CHECK-GI-NEXT:    cmp x13, x1
; CHECK-GI-NEXT:    cset w17, lt
; CHECK-GI-NEXT:    csel w16, w16, w17, eq
; CHECK-GI-NEXT:    cmp x11, x2
; CHECK-GI-NEXT:    cset w17, lo
; CHECK-GI-NEXT:    cmp x12, x3
; CHECK-GI-NEXT:    fmov s0, w16
; CHECK-GI-NEXT:    cset w18, lt
; CHECK-GI-NEXT:    csel w17, w17, w18, eq
; CHECK-GI-NEXT:    cmp x8, x4
; CHECK-GI-NEXT:    cset w18, lo
; CHECK-GI-NEXT:    cmp x9, x5
; CHECK-GI-NEXT:    mov v0.h[1], w17
; CHECK-GI-NEXT:    cset w6, lt
; CHECK-GI-NEXT:    csel w18, w18, w6, eq
; CHECK-GI-NEXT:    cmp x7, #0
; CHECK-GI-NEXT:    cset w6, mi
; CHECK-GI-NEXT:    csel w6, wzr, w6, eq
; CHECK-GI-NEXT:    cmp x14, #0
; CHECK-GI-NEXT:    mov v0.h[2], w18
; CHECK-GI-NEXT:    fmov s1, w6
; CHECK-GI-NEXT:    cset w14, mi
; CHECK-GI-NEXT:    csel w14, wzr, w14, eq
; CHECK-GI-NEXT:    cmp x15, #0
; CHECK-GI-NEXT:    mov v1.h[1], w14
; CHECK-GI-NEXT:    cset w14, mi
; CHECK-GI-NEXT:    csel w14, wzr, w14, eq
; CHECK-GI-NEXT:    mov v1.h[2], w14
; CHECK-GI-NEXT:    eor v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    umov w14, v0.h[0]
; CHECK-GI-NEXT:    umov w15, v0.h[1]
; CHECK-GI-NEXT:    umov w16, v0.h[2]
; CHECK-GI-NEXT:    and w14, w14, #0x1
; CHECK-GI-NEXT:    and w15, w15, #0x1
; CHECK-GI-NEXT:    and w16, w16, #0x1
; CHECK-GI-NEXT:    tst w14, #0x1
; CHECK-GI-NEXT:    csel x0, x10, x0, ne
; CHECK-GI-NEXT:    csel x1, x13, x1, ne
; CHECK-GI-NEXT:    tst w15, #0x1
; CHECK-GI-NEXT:    csel x2, x11, x2, ne
; CHECK-GI-NEXT:    csel x3, x12, x3, ne
; CHECK-GI-NEXT:    tst w16, #0x1
; CHECK-GI-NEXT:    csel x4, x8, x4, ne
; CHECK-GI-NEXT:    csel x5, x9, x5, ne
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i128>, <3 x i1>} @llvm.sadd.with.overflow(<3 x i128> %a, <3 x i128> %b)
  %e0 = extractvalue {<3 x i128>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i128>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i128> %e0, <3 x i128> %a
  ret <3 x i128> %r
}

define <3 x i128> @usub_v3i128(<3 x i128> %a, <3 x i128> %b) {
; CHECK-SD-LABEL: usub_v3i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    ldp x8, x9, [sp]
; CHECK-SD-NEXT:    subs x8, x2, x8
; CHECK-SD-NEXT:    sbcs x9, x3, x9
; CHECK-SD-NEXT:    csel x3, x9, x3, lo
; CHECK-SD-NEXT:    ldp x9, x10, [sp, #16]
; CHECK-SD-NEXT:    csel x2, x8, x2, lo
; CHECK-SD-NEXT:    subs x8, x0, x6
; CHECK-SD-NEXT:    sbcs x11, x1, x7
; CHECK-SD-NEXT:    csel x1, x11, x1, lo
; CHECK-SD-NEXT:    csel x0, x8, x0, lo
; CHECK-SD-NEXT:    subs x8, x4, x9
; CHECK-SD-NEXT:    sbcs x9, x5, x10
; CHECK-SD-NEXT:    csel x4, x8, x4, lo
; CHECK-SD-NEXT:    csel x5, x9, x5, lo
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v3i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    ldp x8, x9, [sp]
; CHECK-GI-NEXT:    subs x10, x0, x6
; CHECK-GI-NEXT:    ldp x11, x12, [sp, #16]
; CHECK-GI-NEXT:    sbc x13, x1, x7
; CHECK-GI-NEXT:    subs x14, x2, x8
; CHECK-GI-NEXT:    sbc x15, x3, x9
; CHECK-GI-NEXT:    subs x16, x4, x11
; CHECK-GI-NEXT:    sbc x17, x5, x12
; CHECK-GI-NEXT:    cmp x0, x6
; CHECK-GI-NEXT:    cset w18, lo
; CHECK-GI-NEXT:    cmp x1, x7
; CHECK-GI-NEXT:    cset w6, lo
; CHECK-GI-NEXT:    csel w18, w18, w6, eq
; CHECK-GI-NEXT:    cmp x2, x8
; CHECK-GI-NEXT:    cset w8, lo
; CHECK-GI-NEXT:    cmp x3, x9
; CHECK-GI-NEXT:    cset w9, lo
; CHECK-GI-NEXT:    csel w8, w8, w9, eq
; CHECK-GI-NEXT:    cmp x4, x11
; CHECK-GI-NEXT:    cset w9, lo
; CHECK-GI-NEXT:    cmp x5, x12
; CHECK-GI-NEXT:    cset w11, lo
; CHECK-GI-NEXT:    csel w9, w9, w11, eq
; CHECK-GI-NEXT:    tst w18, #0x1
; CHECK-GI-NEXT:    csel x0, x10, x0, ne
; CHECK-GI-NEXT:    csel x1, x13, x1, ne
; CHECK-GI-NEXT:    tst w8, #0x1
; CHECK-GI-NEXT:    csel x2, x14, x2, ne
; CHECK-GI-NEXT:    csel x3, x15, x3, ne
; CHECK-GI-NEXT:    tst w9, #0x1
; CHECK-GI-NEXT:    csel x4, x16, x4, ne
; CHECK-GI-NEXT:    csel x5, x17, x5, ne
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i128>, <3 x i1>} @llvm.usub.with.overflow(<3 x i128> %a, <3 x i128> %b)
  %e0 = extractvalue {<3 x i128>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i128>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i128> %e0, <3 x i128> %a
  ret <3 x i128> %r
}

define <3 x i128> @ssub_v3i128(<3 x i128> %a, <3 x i128> %b) {
; CHECK-SD-LABEL: ssub_v3i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    ldp x8, x9, [sp]
; CHECK-SD-NEXT:    subs x8, x2, x8
; CHECK-SD-NEXT:    sbcs x9, x3, x9
; CHECK-SD-NEXT:    csel x3, x9, x3, vs
; CHECK-SD-NEXT:    ldp x9, x10, [sp, #16]
; CHECK-SD-NEXT:    csel x2, x8, x2, vs
; CHECK-SD-NEXT:    subs x8, x0, x6
; CHECK-SD-NEXT:    sbcs x11, x1, x7
; CHECK-SD-NEXT:    csel x1, x11, x1, vs
; CHECK-SD-NEXT:    csel x0, x8, x0, vs
; CHECK-SD-NEXT:    subs x8, x4, x9
; CHECK-SD-NEXT:    sbcs x9, x5, x10
; CHECK-SD-NEXT:    csel x4, x8, x4, vs
; CHECK-SD-NEXT:    csel x5, x9, x5, vs
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v3i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    ldp x14, x15, [sp]
; CHECK-GI-NEXT:    subs x10, x0, x6
; CHECK-GI-NEXT:    ldp x16, x17, [sp, #16]
; CHECK-GI-NEXT:    sbc x13, x1, x7
; CHECK-GI-NEXT:    subs x11, x2, x14
; CHECK-GI-NEXT:    sbc x12, x3, x15
; CHECK-GI-NEXT:    subs x8, x4, x16
; CHECK-GI-NEXT:    sbc x9, x5, x17
; CHECK-GI-NEXT:    cmp x0, x6
; CHECK-GI-NEXT:    cset w18, lo
; CHECK-GI-NEXT:    cmp x1, x7
; CHECK-GI-NEXT:    cset w6, lt
; CHECK-GI-NEXT:    csel w18, w18, w6, eq
; CHECK-GI-NEXT:    cmp x2, x14
; CHECK-GI-NEXT:    cset w14, lo
; CHECK-GI-NEXT:    cmp x3, x15
; CHECK-GI-NEXT:    fmov s0, w18
; CHECK-GI-NEXT:    cset w15, lt
; CHECK-GI-NEXT:    csel w14, w14, w15, eq
; CHECK-GI-NEXT:    cmp x4, x16
; CHECK-GI-NEXT:    cset w15, lo
; CHECK-GI-NEXT:    cmp x5, x17
; CHECK-GI-NEXT:    mov v0.h[1], w14
; CHECK-GI-NEXT:    cset w16, lt
; CHECK-GI-NEXT:    csel w15, w15, w16, eq
; CHECK-GI-NEXT:    cmp x13, #0
; CHECK-GI-NEXT:    cset w16, mi
; CHECK-GI-NEXT:    csel w16, wzr, w16, eq
; CHECK-GI-NEXT:    cmp x12, #0
; CHECK-GI-NEXT:    mov v0.h[2], w15
; CHECK-GI-NEXT:    fmov s1, w16
; CHECK-GI-NEXT:    cset w17, mi
; CHECK-GI-NEXT:    csel w16, wzr, w17, eq
; CHECK-GI-NEXT:    cmp x9, #0
; CHECK-GI-NEXT:    cset w14, mi
; CHECK-GI-NEXT:    mov v1.h[1], w16
; CHECK-GI-NEXT:    csel w14, wzr, w14, eq
; CHECK-GI-NEXT:    mov v1.h[2], w14
; CHECK-GI-NEXT:    eor v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    umov w14, v0.h[0]
; CHECK-GI-NEXT:    umov w15, v0.h[1]
; CHECK-GI-NEXT:    umov w16, v0.h[2]
; CHECK-GI-NEXT:    and w14, w14, #0x1
; CHECK-GI-NEXT:    and w15, w15, #0x1
; CHECK-GI-NEXT:    and w16, w16, #0x1
; CHECK-GI-NEXT:    tst w14, #0x1
; CHECK-GI-NEXT:    csel x0, x10, x0, ne
; CHECK-GI-NEXT:    csel x1, x13, x1, ne
; CHECK-GI-NEXT:    tst w15, #0x1
; CHECK-GI-NEXT:    csel x2, x11, x2, ne
; CHECK-GI-NEXT:    csel x3, x12, x3, ne
; CHECK-GI-NEXT:    tst w16, #0x1
; CHECK-GI-NEXT:    csel x4, x8, x4, ne
; CHECK-GI-NEXT:    csel x5, x9, x5, ne
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i128>, <3 x i1>} @llvm.ssub.with.overflow(<3 x i128> %a, <3 x i128> %b)
  %e0 = extractvalue {<3 x i128>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i128>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i128> %e0, <3 x i128> %a
  ret <3 x i128> %r
}

define <3 x i128> @umul_v3i128(<3 x i128> %a, <3 x i128> %b) {
; CHECK-SD-LABEL: umul_v3i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    stp x20, x19, [sp, #-16]! // 16-byte Folded Spill
; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
; CHECK-SD-NEXT:    .cfi_offset w19, -8
; CHECK-SD-NEXT:    .cfi_offset w20, -16
; CHECK-SD-NEXT:    ldp x8, x9, [sp, #32]
; CHECK-SD-NEXT:    cmp x5, #0
; CHECK-SD-NEXT:    ldp x16, x13, [sp, #16]
; CHECK-SD-NEXT:    umulh x20, x7, x0
; CHECK-SD-NEXT:    mul x11, x9, x4
; CHECK-SD-NEXT:    ccmp x9, #0, #4, ne
; CHECK-SD-NEXT:    umulh x12, x5, x8
; CHECK-SD-NEXT:    umulh x10, x9, x4
; CHECK-SD-NEXT:    madd x11, x5, x8, x11
; CHECK-SD-NEXT:    ccmp xzr, x12, #0, eq
; CHECK-SD-NEXT:    umulh x14, x4, x8
; CHECK-SD-NEXT:    ccmp xzr, x10, #0, eq
; CHECK-SD-NEXT:    mul x17, x13, x2
; CHECK-SD-NEXT:    cset w10, ne
; CHECK-SD-NEXT:    umulh x18, x3, x16
; CHECK-SD-NEXT:    adds x11, x14, x11
; CHECK-SD-NEXT:    umulh x15, x13, x2
; CHECK-SD-NEXT:    csinc w10, w10, wzr, lo
; CHECK-SD-NEXT:    cmp x3, #0
; CHECK-SD-NEXT:    ccmp x13, #0, #4, ne
; CHECK-SD-NEXT:    madd x17, x3, x16, x17
; CHECK-SD-NEXT:    ccmp xzr, x18, #0, eq
; CHECK-SD-NEXT:    umulh x19, x2, x16
; CHECK-SD-NEXT:    ccmp xzr, x15, #0, eq
; CHECK-SD-NEXT:    mul x9, x7, x0
; CHECK-SD-NEXT:    cset w14, ne
; CHECK-SD-NEXT:    umulh x12, x1, x6
; CHECK-SD-NEXT:    adds x15, x19, x17
; CHECK-SD-NEXT:    madd x9, x1, x6, x9
; CHECK-SD-NEXT:    csinc w14, w14, wzr, lo
; CHECK-SD-NEXT:    cmp x1, #0
; CHECK-SD-NEXT:    ccmp x7, #0, #4, ne
; CHECK-SD-NEXT:    umulh x13, x0, x6
; CHECK-SD-NEXT:    ccmp xzr, x12, #0, eq
; CHECK-SD-NEXT:    mul x12, x0, x6
; CHECK-SD-NEXT:    ccmp xzr, x20, #0, eq
; CHECK-SD-NEXT:    cset w17, ne
; CHECK-SD-NEXT:    mul x16, x2, x16
; CHECK-SD-NEXT:    adds x9, x13, x9
; CHECK-SD-NEXT:    mul x8, x4, x8
; CHECK-SD-NEXT:    csinc w13, w17, wzr, lo
; CHECK-SD-NEXT:    cmp w13, #0
; CHECK-SD-NEXT:    csel x0, x12, x0, ne
; CHECK-SD-NEXT:    csel x1, x9, x1, ne
; CHECK-SD-NEXT:    cmp w14, #0
; CHECK-SD-NEXT:    csel x2, x16, x2, ne
; CHECK-SD-NEXT:    csel x3, x15, x3, ne
; CHECK-SD-NEXT:    cmp w10, #0
; CHECK-SD-NEXT:    csel x5, x11, x5, ne
; CHECK-SD-NEXT:    csel x4, x8, x4, ne
; CHECK-SD-NEXT:    ldp x20, x19, [sp], #16 // 16-byte Folded Reload
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v3i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub sp, sp, #128
; CHECK-GI-NEXT:    stp x29, x30, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x28, x27, [sp, #48] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x26, x25, [sp, #64] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x24, x23, [sp, #80] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x22, x21, [sp, #96] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #112] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 128
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    .cfi_offset w23, -40
; CHECK-GI-NEXT:    .cfi_offset w24, -48
; CHECK-GI-NEXT:    .cfi_offset w25, -56
; CHECK-GI-NEXT:    .cfi_offset w26, -64
; CHECK-GI-NEXT:    .cfi_offset w27, -72
; CHECK-GI-NEXT:    .cfi_offset w28, -80
; CHECK-GI-NEXT:    .cfi_offset w30, -88
; CHECK-GI-NEXT:    .cfi_offset w29, -96
; CHECK-GI-NEXT:    umulh x9, x0, xzr
; CHECK-GI-NEXT:    ldp x10, x30, [sp, #128]
; CHECK-GI-NEXT:    ldp x12, x17, [sp, #144]
; CHECK-GI-NEXT:    umulh x13, x0, x6
; CHECK-GI-NEXT:    mul x15, x0, x7
; CHECK-GI-NEXT:    umulh x8, x6, xzr
; CHECK-GI-NEXT:    mul x14, x1, x6
; CHECK-GI-NEXT:    umulh x18, x1, x6
; CHECK-GI-NEXT:    stp x8, x9, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    umulh x19, x0, x7
; CHECK-GI-NEXT:    mul x16, x1, x7
; CHECK-GI-NEXT:    umulh x9, x1, x7
; CHECK-GI-NEXT:    madd x7, x0, x7, x13
; CHECK-GI-NEXT:    adds x13, x13, x15
; CHECK-GI-NEXT:    cset w15, hs
; CHECK-GI-NEXT:    cmn x13, x14
; CHECK-GI-NEXT:    umulh x24, x2, x10
; CHECK-GI-NEXT:    cset w13, hs
; CHECK-GI-NEXT:    adds x14, x18, x19
; CHECK-GI-NEXT:    and x15, x15, #0x1
; CHECK-GI-NEXT:    and x13, x13, #0x1
; CHECK-GI-NEXT:    mul x26, x2, x30
; CHECK-GI-NEXT:    add x13, x15, x13
; CHECK-GI-NEXT:    cset w15, hs
; CHECK-GI-NEXT:    adds x14, x14, x16
; CHECK-GI-NEXT:    mul x25, x3, x10
; CHECK-GI-NEXT:    cset w16, hs
; CHECK-GI-NEXT:    adds x21, x14, x13
; CHECK-GI-NEXT:    cset w14, hs
; CHECK-GI-NEXT:    umulh x8, x2, xzr
; CHECK-GI-NEXT:    and x14, x14, #0x1
; CHECK-GI-NEXT:    umulh x27, x3, x10
; CHECK-GI-NEXT:    umulh x29, x2, x30
; CHECK-GI-NEXT:    stp x8, x9, [sp] // 16-byte Folded Spill
; CHECK-GI-NEXT:    mul x28, x3, x30
; CHECK-GI-NEXT:    umulh x23, x3, x30
; CHECK-GI-NEXT:    madd x30, x2, x30, x24
; CHECK-GI-NEXT:    adds x24, x24, x26
; CHECK-GI-NEXT:    cset w26, hs
; CHECK-GI-NEXT:    cmn x24, x25
; CHECK-GI-NEXT:    umulh x20, x4, x12
; CHECK-GI-NEXT:    cset w24, hs
; CHECK-GI-NEXT:    and x25, x26, #0x1
; CHECK-GI-NEXT:    and x24, x24, #0x1
; CHECK-GI-NEXT:    mul x8, x4, x17
; CHECK-GI-NEXT:    add x24, x25, x24
; CHECK-GI-NEXT:    adds x25, x27, x29
; CHECK-GI-NEXT:    cset w27, hs
; CHECK-GI-NEXT:    adds x25, x25, x28
; CHECK-GI-NEXT:    mul x11, x5, x12
; CHECK-GI-NEXT:    cset w29, hs
; CHECK-GI-NEXT:    adds x24, x25, x24
; CHECK-GI-NEXT:    cset w25, hs
; CHECK-GI-NEXT:    umulh x22, x10, xzr
; CHECK-GI-NEXT:    adds x8, x20, x8
; CHECK-GI-NEXT:    cset w9, hs
; CHECK-GI-NEXT:    umulh x18, x5, x12
; CHECK-GI-NEXT:    cmn x8, x11
; CHECK-GI-NEXT:    and x8, x15, #0x1
; CHECK-GI-NEXT:    and x11, x16, #0x1
; CHECK-GI-NEXT:    add x8, x8, x11
; CHECK-GI-NEXT:    ldp x16, x11, [sp, #8] // 16-byte Folded Reload
; CHECK-GI-NEXT:    add x8, x8, x14
; CHECK-GI-NEXT:    ldr x14, [sp, #24] // 8-byte Reload
; CHECK-GI-NEXT:    umulh x19, x4, x17
; CHECK-GI-NEXT:    and x9, x9, #0x1
; CHECK-GI-NEXT:    add x11, x11, x16
; CHECK-GI-NEXT:    mul x26, x5, x17
; CHECK-GI-NEXT:    and x16, x29, #0x1
; CHECK-GI-NEXT:    add x11, x11, x14
; CHECK-GI-NEXT:    add x8, x11, x8
; CHECK-GI-NEXT:    and x11, x27, #0x1
; CHECK-GI-NEXT:    umulh x28, x12, xzr
; CHECK-GI-NEXT:    add x11, x11, x16
; CHECK-GI-NEXT:    and x16, x25, #0x1
; CHECK-GI-NEXT:    orr x8, x21, x8
; CHECK-GI-NEXT:    umulh x13, x5, x17
; CHECK-GI-NEXT:    add x11, x11, x16
; CHECK-GI-NEXT:    add x16, x22, x23
; CHECK-GI-NEXT:    cset w22, hs
; CHECK-GI-NEXT:    adds x18, x18, x19
; CHECK-GI-NEXT:    ldr x23, [sp] // 8-byte Reload
; CHECK-GI-NEXT:    umulh x15, x4, xzr
; CHECK-GI-NEXT:    and x22, x22, #0x1
; CHECK-GI-NEXT:    cset w19, hs
; CHECK-GI-NEXT:    add x9, x9, x22
; CHECK-GI-NEXT:    adds x18, x18, x26
; CHECK-GI-NEXT:    and x19, x19, #0x1
; CHECK-GI-NEXT:    madd x7, x1, x6, x7
; CHECK-GI-NEXT:    add x16, x16, x23
; CHECK-GI-NEXT:    add x13, x28, x13
; CHECK-GI-NEXT:    add x11, x16, x11
; CHECK-GI-NEXT:    mul x14, x0, x6
; CHECK-GI-NEXT:    orr x11, x24, x11
; CHECK-GI-NEXT:    add x13, x13, x15
; CHECK-GI-NEXT:    ldp x24, x23, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT:    madd x17, x4, x17, x20
; CHECK-GI-NEXT:    cset w20, hs
; CHECK-GI-NEXT:    adds x9, x18, x9
; CHECK-GI-NEXT:    and x18, x20, #0x1
; CHECK-GI-NEXT:    cset w20, hs
; CHECK-GI-NEXT:    cmp x8, #0
; CHECK-GI-NEXT:    madd x6, x3, x10, x30
; CHECK-GI-NEXT:    add x18, x19, x18
; CHECK-GI-NEXT:    and x19, x20, #0x1
; CHECK-GI-NEXT:    add x18, x18, x19
; CHECK-GI-NEXT:    csel x0, x14, x0, ne
; CHECK-GI-NEXT:    csel x1, x7, x1, ne
; CHECK-GI-NEXT:    mul x10, x2, x10
; CHECK-GI-NEXT:    add x13, x13, x18
; CHECK-GI-NEXT:    cmp x11, #0
; CHECK-GI-NEXT:    orr x9, x9, x13
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mul x22, x4, x12
; CHECK-GI-NEXT:    ldp x26, x25, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x3, x6, x3, ne
; CHECK-GI-NEXT:    ldp x28, x27, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT:    madd x12, x5, x12, x17
; CHECK-GI-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x2, x10, x2, ne
; CHECK-GI-NEXT:    cmp x9, #0
; CHECK-GI-NEXT:    csel x4, x22, x4, ne
; CHECK-GI-NEXT:    ldp x22, x21, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x5, x12, x5, ne
; CHECK-GI-NEXT:    add sp, sp, #128
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i128>, <3 x i1>} @llvm.umul.with.overflow(<3 x i128> %a, <3 x i128> %b)
  %e0 = extractvalue {<3 x i128>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i128>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i128> %e0, <3 x i128> %a
  ret <3 x i128> %r
}

define <3 x i128> @smul_v3i128(<3 x i128> %a, <3 x i128> %b) {
; CHECK-SD-LABEL: smul_v3i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    str x29, [sp, #-96]! // 8-byte Folded Spill
; CHECK-SD-NEXT:    stp x28, x27, [sp, #16] // 16-byte Folded Spill
; CHECK-SD-NEXT:    stp x26, x25, [sp, #32] // 16-byte Folded Spill
; CHECK-SD-NEXT:    stp x24, x23, [sp, #48] // 16-byte Folded Spill
; CHECK-SD-NEXT:    stp x22, x21, [sp, #64] // 16-byte Folded Spill
; CHECK-SD-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
; CHECK-SD-NEXT:    .cfi_def_cfa_offset 96
; CHECK-SD-NEXT:    .cfi_offset w19, -8
; CHECK-SD-NEXT:    .cfi_offset w20, -16
; CHECK-SD-NEXT:    .cfi_offset w21, -24
; CHECK-SD-NEXT:    .cfi_offset w22, -32
; CHECK-SD-NEXT:    .cfi_offset w23, -40
; CHECK-SD-NEXT:    .cfi_offset w24, -48
; CHECK-SD-NEXT:    .cfi_offset w25, -56
; CHECK-SD-NEXT:    .cfi_offset w26, -64
; CHECK-SD-NEXT:    .cfi_offset w27, -72
; CHECK-SD-NEXT:    .cfi_offset w28, -80
; CHECK-SD-NEXT:    .cfi_offset w29, -96
; CHECK-SD-NEXT:    ldp x8, x13, [sp, #112]
; CHECK-SD-NEXT:    asr x9, x5, #63
; CHECK-SD-NEXT:    asr x20, x3, #63
; CHECK-SD-NEXT:    asr x28, x1, #63
; CHECK-SD-NEXT:    umulh x29, x1, x6
; CHECK-SD-NEXT:    mul x11, x9, x8
; CHECK-SD-NEXT:    asr x9, x13, #63
; CHECK-SD-NEXT:    umulh x12, x4, x8
; CHECK-SD-NEXT:    mul x14, x5, x8
; CHECK-SD-NEXT:    umulh x10, x5, x8
; CHECK-SD-NEXT:    mul x17, x4, x13
; CHECK-SD-NEXT:    adds x12, x14, x12
; CHECK-SD-NEXT:    umulh x15, x4, x13
; CHECK-SD-NEXT:    adc x11, x10, x11
; CHECK-SD-NEXT:    mul x16, x4, x9
; CHECK-SD-NEXT:    ldp x9, x22, [sp, #96]
; CHECK-SD-NEXT:    adds x10, x17, x12
; CHECK-SD-NEXT:    smulh x18, x5, x13
; CHECK-SD-NEXT:    asr x25, x22, #63
; CHECK-SD-NEXT:    mul x13, x5, x13
; CHECK-SD-NEXT:    adc x12, x15, x16
; CHECK-SD-NEXT:    asr x16, x11, #63
; CHECK-SD-NEXT:    umulh x21, x2, x9
; CHECK-SD-NEXT:    asr x17, x12, #63
; CHECK-SD-NEXT:    adds x11, x11, x12
; CHECK-SD-NEXT:    mul x23, x3, x9
; CHECK-SD-NEXT:    adc x16, x16, x17
; CHECK-SD-NEXT:    adds x11, x13, x11
; CHECK-SD-NEXT:    umulh x19, x3, x9
; CHECK-SD-NEXT:    adc x13, x18, x16
; CHECK-SD-NEXT:    mul x20, x20, x9
; CHECK-SD-NEXT:    adds x16, x23, x21
; CHECK-SD-NEXT:    mul x26, x2, x22
; CHECK-SD-NEXT:    umulh x24, x2, x22
; CHECK-SD-NEXT:    adc x18, x19, x20
; CHECK-SD-NEXT:    asr x19, x7, #63
; CHECK-SD-NEXT:    mul x25, x2, x25
; CHECK-SD-NEXT:    asr x23, x18, #63
; CHECK-SD-NEXT:    adds x16, x26, x16
; CHECK-SD-NEXT:    smulh x27, x3, x22
; CHECK-SD-NEXT:    mul x22, x3, x22
; CHECK-SD-NEXT:    adc x21, x24, x25
; CHECK-SD-NEXT:    ldp x26, x25, [sp, #32] // 16-byte Folded Reload
; CHECK-SD-NEXT:    umulh x15, x0, x6
; CHECK-SD-NEXT:    asr x24, x21, #63
; CHECK-SD-NEXT:    adds x18, x18, x21
; CHECK-SD-NEXT:    mul x12, x1, x6
; CHECK-SD-NEXT:    adc x21, x23, x24
; CHECK-SD-NEXT:    adds x18, x22, x18
; CHECK-SD-NEXT:    ldp x24, x23, [sp, #48] // 16-byte Folded Reload
; CHECK-SD-NEXT:    mul x14, x28, x6
; CHECK-SD-NEXT:    adc x21, x27, x21
; CHECK-SD-NEXT:    ldp x28, x27, [sp, #16] // 16-byte Folded Reload
; CHECK-SD-NEXT:    mul x20, x0, x7
; CHECK-SD-NEXT:    adds x12, x12, x15
; CHECK-SD-NEXT:    umulh x17, x0, x7
; CHECK-SD-NEXT:    adc x14, x29, x14
; CHECK-SD-NEXT:    mul x19, x0, x19
; CHECK-SD-NEXT:    adds x12, x20, x12
; CHECK-SD-NEXT:    asr x20, x14, #63
; CHECK-SD-NEXT:    mul x15, x1, x7
; CHECK-SD-NEXT:    smulh x7, x1, x7
; CHECK-SD-NEXT:    adc x17, x17, x19
; CHECK-SD-NEXT:    asr x19, x10, #63
; CHECK-SD-NEXT:    asr x22, x17, #63
; CHECK-SD-NEXT:    adds x14, x14, x17
; CHECK-SD-NEXT:    mul x6, x0, x6
; CHECK-SD-NEXT:    eor x13, x13, x19
; CHECK-SD-NEXT:    eor x11, x11, x19
; CHECK-SD-NEXT:    adc x17, x20, x22
; CHECK-SD-NEXT:    adds x14, x15, x14
; CHECK-SD-NEXT:    asr x15, x12, #63
; CHECK-SD-NEXT:    mul x9, x2, x9
; CHECK-SD-NEXT:    adc x17, x7, x17
; CHECK-SD-NEXT:    asr x7, x16, #63
; CHECK-SD-NEXT:    eor x17, x17, x15
; CHECK-SD-NEXT:    eor x14, x14, x15
; CHECK-SD-NEXT:    orr x11, x11, x13
; CHECK-SD-NEXT:    mul x8, x4, x8
; CHECK-SD-NEXT:    eor x15, x21, x7
; CHECK-SD-NEXT:    eor x18, x18, x7
; CHECK-SD-NEXT:    orr x14, x14, x17
; CHECK-SD-NEXT:    orr x13, x18, x15
; CHECK-SD-NEXT:    cmp x14, #0
; CHECK-SD-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
; CHECK-SD-NEXT:    csel x0, x6, x0, ne
; CHECK-SD-NEXT:    csel x1, x12, x1, ne
; CHECK-SD-NEXT:    cmp x13, #0
; CHECK-SD-NEXT:    ldp x22, x21, [sp, #64] // 16-byte Folded Reload
; CHECK-SD-NEXT:    csel x2, x9, x2, ne
; CHECK-SD-NEXT:    csel x3, x16, x3, ne
; CHECK-SD-NEXT:    cmp x11, #0
; CHECK-SD-NEXT:    csel x4, x8, x4, ne
; CHECK-SD-NEXT:    csel x5, x10, x5, ne
; CHECK-SD-NEXT:    ldr x29, [sp], #96 // 8-byte Folded Reload
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v3i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub sp, sp, #128
; CHECK-GI-NEXT:    stp x29, x30, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x28, x27, [sp, #48] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x26, x25, [sp, #64] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x24, x23, [sp, #80] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x22, x21, [sp, #96] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #112] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 128
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    .cfi_offset w23, -40
; CHECK-GI-NEXT:    .cfi_offset w24, -48
; CHECK-GI-NEXT:    .cfi_offset w25, -56
; CHECK-GI-NEXT:    .cfi_offset w26, -64
; CHECK-GI-NEXT:    .cfi_offset w27, -72
; CHECK-GI-NEXT:    .cfi_offset w28, -80
; CHECK-GI-NEXT:    .cfi_offset w30, -88
; CHECK-GI-NEXT:    .cfi_offset w29, -96
; CHECK-GI-NEXT:    mul x9, x0, x7
; CHECK-GI-NEXT:    asr x12, x1, #63
; CHECK-GI-NEXT:    asr x13, x7, #63
; CHECK-GI-NEXT:    ldp x10, x21, [sp, #128]
; CHECK-GI-NEXT:    mul x8, x1, x6
; CHECK-GI-NEXT:    umulh x15, x1, x7
; CHECK-GI-NEXT:    asr x22, x21, #63
; CHECK-GI-NEXT:    umulh x16, x0, x13
; CHECK-GI-NEXT:    stp x9, x8, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    umulh x8, x12, x6
; CHECK-GI-NEXT:    umulh x23, x3, x21
; CHECK-GI-NEXT:    umulh x24, x2, x22
; CHECK-GI-NEXT:    add x8, x8, x15
; CHECK-GI-NEXT:    asr x15, x3, #63
; CHECK-GI-NEXT:    add x8, x8, x16
; CHECK-GI-NEXT:    mul x19, x0, x13
; CHECK-GI-NEXT:    madd x8, x0, x13, x8
; CHECK-GI-NEXT:    umulh x17, x15, x10
; CHECK-GI-NEXT:    madd x8, x1, x13, x8
; CHECK-GI-NEXT:    mul x16, x12, x6
; CHECK-GI-NEXT:    madd x13, x12, x7, x8
; CHECK-GI-NEXT:    umulh x11, x0, x6
; CHECK-GI-NEXT:    madd x9, x12, x6, x13
; CHECK-GI-NEXT:    add x12, x17, x23
; CHECK-GI-NEXT:    add x12, x12, x24
; CHECK-GI-NEXT:    madd x12, x2, x22, x12
; CHECK-GI-NEXT:    mul x24, x2, x22
; CHECK-GI-NEXT:    madd x12, x3, x22, x12
; CHECK-GI-NEXT:    asr x22, x5, #63
; CHECK-GI-NEXT:    umulh x14, x1, x6
; CHECK-GI-NEXT:    madd x12, x15, x21, x12
; CHECK-GI-NEXT:    umulh x20, x0, x7
; CHECK-GI-NEXT:    madd x8, x15, x10, x12
; CHECK-GI-NEXT:    ldp x12, x17, [sp, #144]
; CHECK-GI-NEXT:    mul x18, x1, x7
; CHECK-GI-NEXT:    asr x28, x17, #63
; CHECK-GI-NEXT:    umulh x27, x22, x12
; CHECK-GI-NEXT:    stp x8, x9, [sp] // 16-byte Folded Spill
; CHECK-GI-NEXT:    ldp x8, x13, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT:    umulh x29, x5, x17
; CHECK-GI-NEXT:    umulh x30, x4, x28
; CHECK-GI-NEXT:    madd x7, x0, x7, x11
; CHECK-GI-NEXT:    adds x11, x11, x8
; CHECK-GI-NEXT:    cset w8, hs
; CHECK-GI-NEXT:    cmn x11, x13
; CHECK-GI-NEXT:    add x27, x27, x29
; CHECK-GI-NEXT:    umulh x25, x2, x10
; CHECK-GI-NEXT:    cset w11, hs
; CHECK-GI-NEXT:    adds x14, x14, x20
; CHECK-GI-NEXT:    cset w20, hs
; CHECK-GI-NEXT:    adds x14, x14, x19
; CHECK-GI-NEXT:    add x27, x27, x30
; CHECK-GI-NEXT:    mul x26, x2, x21
; CHECK-GI-NEXT:    and x8, x8, #0x1
; CHECK-GI-NEXT:    and x20, x20, #0x1
; CHECK-GI-NEXT:    mul x9, x3, x10
; CHECK-GI-NEXT:    mul x23, x15, x10
; CHECK-GI-NEXT:    umulh x15, x3, x10
; CHECK-GI-NEXT:    umulh x29, x2, x21
; CHECK-GI-NEXT:    mul x30, x3, x21
; CHECK-GI-NEXT:    madd x19, x2, x21, x25
; CHECK-GI-NEXT:    cset w21, hs
; CHECK-GI-NEXT:    adds x14, x14, x18
; CHECK-GI-NEXT:    and x18, x11, #0x1
; CHECK-GI-NEXT:    and x21, x21, #0x1
; CHECK-GI-NEXT:    madd x27, x4, x28, x27
; CHECK-GI-NEXT:    add x8, x8, x18
; CHECK-GI-NEXT:    cset w18, hs
; CHECK-GI-NEXT:    adds x14, x14, x16
; CHECK-GI-NEXT:    add x20, x20, x21
; CHECK-GI-NEXT:    and x18, x18, #0x1
; CHECK-GI-NEXT:    cset w16, hs
; CHECK-GI-NEXT:    adds x8, x14, x8
; CHECK-GI-NEXT:    umulh x13, x4, x12
; CHECK-GI-NEXT:    cset w11, hs
; CHECK-GI-NEXT:    adds x25, x25, x26
; CHECK-GI-NEXT:    str x8, [sp, #24] // 8-byte Spill
; CHECK-GI-NEXT:    madd x27, x5, x28, x27
; CHECK-GI-NEXT:    cset w26, hs
; CHECK-GI-NEXT:    cmn x25, x9
; CHECK-GI-NEXT:    cset w25, hs
; CHECK-GI-NEXT:    adds x15, x15, x29
; CHECK-GI-NEXT:    add x18, x20, x18
; CHECK-GI-NEXT:    mul x8, x4, x17
; CHECK-GI-NEXT:    and x25, x25, #0x1
; CHECK-GI-NEXT:    and x16, x16, #0x1
; CHECK-GI-NEXT:    and x11, x11, #0x1
; CHECK-GI-NEXT:    mul x14, x4, x28
; CHECK-GI-NEXT:    cset w28, hs
; CHECK-GI-NEXT:    adds x15, x15, x24
; CHECK-GI-NEXT:    madd x24, x22, x17, x27
; CHECK-GI-NEXT:    cset w27, hs
; CHECK-GI-NEXT:    adds x15, x15, x30
; CHECK-GI-NEXT:    cset w29, hs
; CHECK-GI-NEXT:    adds x15, x15, x23
; CHECK-GI-NEXT:    and x23, x26, #0x1
; CHECK-GI-NEXT:    mul x30, x5, x12
; CHECK-GI-NEXT:    add x21, x23, x25
; CHECK-GI-NEXT:    cset w23, hs
; CHECK-GI-NEXT:    adds x15, x15, x21
; CHECK-GI-NEXT:    cset w20, hs
; CHECK-GI-NEXT:    adds x8, x13, x8
; CHECK-GI-NEXT:    umulh x26, x5, x12
; CHECK-GI-NEXT:    cset w25, hs
; CHECK-GI-NEXT:    umulh x21, x4, x17
; CHECK-GI-NEXT:    cmn x8, x30
; CHECK-GI-NEXT:    add x8, x18, x16
; CHECK-GI-NEXT:    and x16, x28, #0x1
; CHECK-GI-NEXT:    and x18, x27, #0x1
; CHECK-GI-NEXT:    add x8, x8, x11
; CHECK-GI-NEXT:    mul x9, x5, x17
; CHECK-GI-NEXT:    add x11, x16, x18
; CHECK-GI-NEXT:    and x16, x29, #0x1
; CHECK-GI-NEXT:    add x11, x11, x16
; CHECK-GI-NEXT:    and x16, x23, #0x1
; CHECK-GI-NEXT:    mul x18, x22, x12
; CHECK-GI-NEXT:    add x11, x11, x16
; CHECK-GI-NEXT:    and x16, x20, #0x1
; CHECK-GI-NEXT:    cset w20, hs
; CHECK-GI-NEXT:    madd x24, x22, x12, x24
; CHECK-GI-NEXT:    and x22, x25, #0x1
; CHECK-GI-NEXT:    and x20, x20, #0x1
; CHECK-GI-NEXT:    add x11, x11, x16
; CHECK-GI-NEXT:    add x16, x22, x20
; CHECK-GI-NEXT:    adds x20, x26, x21
; CHECK-GI-NEXT:    cset w21, hs
; CHECK-GI-NEXT:    adds x14, x20, x14
; CHECK-GI-NEXT:    madd x13, x4, x17, x13
; CHECK-GI-NEXT:    cset w20, hs
; CHECK-GI-NEXT:    adds x9, x14, x9
; CHECK-GI-NEXT:    and x17, x21, #0x1
; CHECK-GI-NEXT:    and x14, x20, #0x1
; CHECK-GI-NEXT:    cset w20, hs
; CHECK-GI-NEXT:    adds x9, x9, x18
; CHECK-GI-NEXT:    add x14, x17, x14
; CHECK-GI-NEXT:    and x17, x20, #0x1
; CHECK-GI-NEXT:    cset w18, hs
; CHECK-GI-NEXT:    adds x9, x9, x16
; CHECK-GI-NEXT:    madd x7, x1, x6, x7
; CHECK-GI-NEXT:    add x14, x14, x17
; CHECK-GI-NEXT:    and x16, x18, #0x1
; CHECK-GI-NEXT:    cset w17, hs
; CHECK-GI-NEXT:    madd x19, x3, x10, x19
; CHECK-GI-NEXT:    add x14, x14, x16
; CHECK-GI-NEXT:    and x16, x17, #0x1
; CHECK-GI-NEXT:    add x14, x14, x16
; CHECK-GI-NEXT:    ldr x16, [sp, #24] // 8-byte Reload
; CHECK-GI-NEXT:    mul x17, x0, x6
; CHECK-GI-NEXT:    ldp x18, x6, [sp] // 16-byte Folded Reload
; CHECK-GI-NEXT:    eor x16, x16, x7, asr #63
; CHECK-GI-NEXT:    add x14, x24, x14
; CHECK-GI-NEXT:    madd x13, x5, x12, x13
; CHECK-GI-NEXT:    ldp x22, x21, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT:    add x8, x6, x8
; CHECK-GI-NEXT:    add x11, x18, x11
; CHECK-GI-NEXT:    eor x15, x15, x19, asr #63
; CHECK-GI-NEXT:    eor x8, x8, x7, asr #63
; CHECK-GI-NEXT:    mul x10, x2, x10
; CHECK-GI-NEXT:    eor x11, x11, x19, asr #63
; CHECK-GI-NEXT:    ldp x24, x23, [sp, #80] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mul x12, x4, x12
; CHECK-GI-NEXT:    orr x8, x16, x8
; CHECK-GI-NEXT:    orr x11, x15, x11
; CHECK-GI-NEXT:    eor x9, x9, x13, asr #63
; CHECK-GI-NEXT:    eor x14, x14, x13, asr #63
; CHECK-GI-NEXT:    cmp x8, #0
; CHECK-GI-NEXT:    csel x0, x17, x0, ne
; CHECK-GI-NEXT:    csel x1, x7, x1, ne
; CHECK-GI-NEXT:    cmp x11, #0
; CHECK-GI-NEXT:    orr x9, x9, x14
; CHECK-GI-NEXT:    csel x3, x19, x3, ne
; CHECK-GI-NEXT:    csel x2, x10, x2, ne
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT:    cmp x9, #0
; CHECK-GI-NEXT:    ldp x26, x25, [sp, #64] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x4, x12, x4, ne
; CHECK-GI-NEXT:    ldp x28, x27, [sp, #48] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x5, x13, x5, ne
; CHECK-GI-NEXT:    ldp x29, x30, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    add sp, sp, #128
; CHECK-GI-NEXT:    ret
  %o = call {<3 x i128>, <3 x i1>} @llvm.smul.with.overflow(<3 x i128> %a, <3 x i128> %b)
  %e0 = extractvalue {<3 x i128>, <3 x i1>} %o, 0
  %e1 = extractvalue {<3 x i128>, <3 x i1>} %o, 1
  %r = select <3 x i1> %e1, <3 x i128> %e0, <3 x i128> %a
  ret <3 x i128> %r
}

define <4 x i128> @uadd_v4i128(<4 x i128> %a, <4 x i128> %b) {
; CHECK-SD-LABEL: uadd_v4i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    ldp x8, x9, [sp, #16]
; CHECK-SD-NEXT:    ldp x10, x11, [sp, #32]
; CHECK-SD-NEXT:    adds x8, x2, x8
; CHECK-SD-NEXT:    adcs x9, x3, x9
; CHECK-SD-NEXT:    csel x3, x9, x3, hs
; CHECK-SD-NEXT:    csel x2, x8, x2, hs
; CHECK-SD-NEXT:    adds x8, x4, x10
; CHECK-SD-NEXT:    adcs x9, x5, x11
; CHECK-SD-NEXT:    ldp x10, x11, [sp]
; CHECK-SD-NEXT:    csel x4, x8, x4, hs
; CHECK-SD-NEXT:    csel x5, x9, x5, hs
; CHECK-SD-NEXT:    adds x8, x0, x10
; CHECK-SD-NEXT:    ldp x9, x10, [sp, #48]
; CHECK-SD-NEXT:    adcs x11, x1, x11
; CHECK-SD-NEXT:    csel x1, x11, x1, hs
; CHECK-SD-NEXT:    csel x0, x8, x0, hs
; CHECK-SD-NEXT:    adds x8, x6, x9
; CHECK-SD-NEXT:    adcs x9, x7, x10
; CHECK-SD-NEXT:    csel x6, x8, x6, hs
; CHECK-SD-NEXT:    csel x7, x9, x7, hs
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uadd_v4i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    str x23, [sp, #-48]! // 8-byte Folded Spill
; CHECK-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    .cfi_offset w23, -48
; CHECK-GI-NEXT:    ldp x8, x9, [sp, #48]
; CHECK-GI-NEXT:    ldp x10, x11, [sp, #64]
; CHECK-GI-NEXT:    ldp x12, x13, [sp, #80]
; CHECK-GI-NEXT:    adds x14, x0, x8
; CHECK-GI-NEXT:    ldp x17, x18, [sp, #96]
; CHECK-GI-NEXT:    adc x15, x1, x9
; CHECK-GI-NEXT:    adds x16, x2, x10
; CHECK-GI-NEXT:    adc x19, x3, x11
; CHECK-GI-NEXT:    adds x20, x4, x12
; CHECK-GI-NEXT:    adc x21, x5, x13
; CHECK-GI-NEXT:    adds x22, x6, x17
; CHECK-GI-NEXT:    adc x23, x7, x18
; CHECK-GI-NEXT:    cmp x14, x8
; CHECK-GI-NEXT:    cset w8, lo
; CHECK-GI-NEXT:    cmp x15, x9
; CHECK-GI-NEXT:    cset w9, lo
; CHECK-GI-NEXT:    csel w8, w8, w9, eq
; CHECK-GI-NEXT:    cmp x16, x10
; CHECK-GI-NEXT:    cset w9, lo
; CHECK-GI-NEXT:    cmp x19, x11
; CHECK-GI-NEXT:    cset w10, lo
; CHECK-GI-NEXT:    csel w9, w9, w10, eq
; CHECK-GI-NEXT:    cmp x20, x12
; CHECK-GI-NEXT:    cset w10, lo
; CHECK-GI-NEXT:    cmp x21, x13
; CHECK-GI-NEXT:    cset w11, lo
; CHECK-GI-NEXT:    csel w10, w10, w11, eq
; CHECK-GI-NEXT:    cmp x22, x17
; CHECK-GI-NEXT:    cset w11, lo
; CHECK-GI-NEXT:    cmp x23, x18
; CHECK-GI-NEXT:    cset w12, lo
; CHECK-GI-NEXT:    csel w11, w11, w12, eq
; CHECK-GI-NEXT:    tst w8, #0x1
; CHECK-GI-NEXT:    csel x0, x14, x0, ne
; CHECK-GI-NEXT:    csel x1, x15, x1, ne
; CHECK-GI-NEXT:    tst w9, #0x1
; CHECK-GI-NEXT:    csel x2, x16, x2, ne
; CHECK-GI-NEXT:    csel x3, x19, x3, ne
; CHECK-GI-NEXT:    tst w10, #0x1
; CHECK-GI-NEXT:    csel x4, x20, x4, ne
; CHECK-GI-NEXT:    csel x5, x21, x5, ne
; CHECK-GI-NEXT:    tst w11, #0x1
; CHECK-GI-NEXT:    csel x6, x22, x6, ne
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x7, x23, x7, ne
; CHECK-GI-NEXT:    ldr x23, [sp], #48 // 8-byte Folded Reload
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i128>, <4 x i1>} @llvm.uadd.with.overflow(<4 x i128> %a, <4 x i128> %b)
  %e0 = extractvalue {<4 x i128>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i128>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i128> %e0, <4 x i128> %a
  ret <4 x i128> %r
}

define <4 x i128> @sadd_v4i128(<4 x i128> %a, <4 x i128> %b) {
; CHECK-SD-LABEL: sadd_v4i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    ldp x8, x9, [sp, #16]
; CHECK-SD-NEXT:    ldp x10, x11, [sp, #32]
; CHECK-SD-NEXT:    adds x8, x2, x8
; CHECK-SD-NEXT:    adcs x9, x3, x9
; CHECK-SD-NEXT:    csel x3, x9, x3, vs
; CHECK-SD-NEXT:    csel x2, x8, x2, vs
; CHECK-SD-NEXT:    adds x8, x4, x10
; CHECK-SD-NEXT:    adcs x9, x5, x11
; CHECK-SD-NEXT:    ldp x10, x11, [sp]
; CHECK-SD-NEXT:    csel x4, x8, x4, vs
; CHECK-SD-NEXT:    csel x5, x9, x5, vs
; CHECK-SD-NEXT:    adds x8, x0, x10
; CHECK-SD-NEXT:    ldp x9, x10, [sp, #48]
; CHECK-SD-NEXT:    adcs x11, x1, x11
; CHECK-SD-NEXT:    csel x1, x11, x1, vs
; CHECK-SD-NEXT:    csel x0, x8, x0, vs
; CHECK-SD-NEXT:    adds x8, x6, x9
; CHECK-SD-NEXT:    adcs x9, x7, x10
; CHECK-SD-NEXT:    csel x6, x8, x6, vs
; CHECK-SD-NEXT:    csel x7, x9, x7, vs
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sadd_v4i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    stp x24, x23, [sp, #-48]! // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    .cfi_offset w23, -40
; CHECK-GI-NEXT:    .cfi_offset w24, -48
; CHECK-GI-NEXT:    ldp x8, x18, [sp, #48]
; CHECK-GI-NEXT:    ldp x9, x19, [sp, #64]
; CHECK-GI-NEXT:    ldp x11, x16, [sp, #80]
; CHECK-GI-NEXT:    adds x13, x0, x8
; CHECK-GI-NEXT:    ldp x8, x17, [sp, #96]
; CHECK-GI-NEXT:    adc x14, x1, x18
; CHECK-GI-NEXT:    adds x10, x2, x9
; CHECK-GI-NEXT:    adc x15, x3, x19
; CHECK-GI-NEXT:    adds x11, x4, x11
; CHECK-GI-NEXT:    adc x12, x5, x16
; CHECK-GI-NEXT:    adds x8, x6, x8
; CHECK-GI-NEXT:    adc x9, x7, x17
; CHECK-GI-NEXT:    cmp x13, x0
; CHECK-GI-NEXT:    cset w20, lo
; CHECK-GI-NEXT:    cmp x14, x1
; CHECK-GI-NEXT:    cset w21, lt
; CHECK-GI-NEXT:    csel w20, w20, w21, eq
; CHECK-GI-NEXT:    cmp x10, x2
; CHECK-GI-NEXT:    cset w21, lo
; CHECK-GI-NEXT:    cmp x15, x3
; CHECK-GI-NEXT:    fmov s0, w20
; CHECK-GI-NEXT:    cset w22, lt
; CHECK-GI-NEXT:    csel w21, w21, w22, eq
; CHECK-GI-NEXT:    cmp x11, x4
; CHECK-GI-NEXT:    cset w22, lo
; CHECK-GI-NEXT:    cmp x12, x5
; CHECK-GI-NEXT:    mov v0.h[1], w21
; CHECK-GI-NEXT:    cset w23, lt
; CHECK-GI-NEXT:    csel w22, w22, w23, eq
; CHECK-GI-NEXT:    cmp x8, x6
; CHECK-GI-NEXT:    cset w23, lo
; CHECK-GI-NEXT:    cmp x9, x7
; CHECK-GI-NEXT:    cset w24, lt
; CHECK-GI-NEXT:    mov v0.h[2], w22
; CHECK-GI-NEXT:    csel w23, w23, w24, eq
; CHECK-GI-NEXT:    cmp x18, #0
; CHECK-GI-NEXT:    cset w18, mi
; CHECK-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel w18, wzr, w18, eq
; CHECK-GI-NEXT:    cmp x19, #0
; CHECK-GI-NEXT:    fmov s1, w18
; CHECK-GI-NEXT:    cset w19, mi
; CHECK-GI-NEXT:    mov v0.h[3], w23
; CHECK-GI-NEXT:    csel w18, wzr, w19, eq
; CHECK-GI-NEXT:    cmp x16, #0
; CHECK-GI-NEXT:    cset w16, mi
; CHECK-GI-NEXT:    mov v1.h[1], w18
; CHECK-GI-NEXT:    csel w16, wzr, w16, eq
; CHECK-GI-NEXT:    cmp x17, #0
; CHECK-GI-NEXT:    mov v1.h[2], w16
; CHECK-GI-NEXT:    cset w16, mi
; CHECK-GI-NEXT:    csel w16, wzr, w16, eq
; CHECK-GI-NEXT:    mov v1.h[3], w16
; CHECK-GI-NEXT:    eor v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    umov w16, v0.h[0]
; CHECK-GI-NEXT:    umov w17, v0.h[1]
; CHECK-GI-NEXT:    umov w18, v0.h[2]
; CHECK-GI-NEXT:    umov w19, v0.h[3]
; CHECK-GI-NEXT:    and w16, w16, #0x1
; CHECK-GI-NEXT:    and w17, w17, #0x1
; CHECK-GI-NEXT:    and w18, w18, #0x1
; CHECK-GI-NEXT:    tst w16, #0x1
; CHECK-GI-NEXT:    and w19, w19, #0x1
; CHECK-GI-NEXT:    csel x0, x13, x0, ne
; CHECK-GI-NEXT:    csel x1, x14, x1, ne
; CHECK-GI-NEXT:    tst w17, #0x1
; CHECK-GI-NEXT:    csel x2, x10, x2, ne
; CHECK-GI-NEXT:    csel x3, x15, x3, ne
; CHECK-GI-NEXT:    tst w18, #0x1
; CHECK-GI-NEXT:    csel x4, x11, x4, ne
; CHECK-GI-NEXT:    csel x5, x12, x5, ne
; CHECK-GI-NEXT:    tst w19, #0x1
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x6, x8, x6, ne
; CHECK-GI-NEXT:    csel x7, x9, x7, ne
; CHECK-GI-NEXT:    ldp x24, x23, [sp], #48 // 16-byte Folded Reload
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i128>, <4 x i1>} @llvm.sadd.with.overflow(<4 x i128> %a, <4 x i128> %b)
  %e0 = extractvalue {<4 x i128>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i128>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i128> %e0, <4 x i128> %a
  ret <4 x i128> %r
}

define <4 x i128> @usub_v4i128(<4 x i128> %a, <4 x i128> %b) {
; CHECK-SD-LABEL: usub_v4i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    ldp x8, x9, [sp, #16]
; CHECK-SD-NEXT:    ldp x10, x11, [sp, #32]
; CHECK-SD-NEXT:    subs x8, x2, x8
; CHECK-SD-NEXT:    sbcs x9, x3, x9
; CHECK-SD-NEXT:    csel x3, x9, x3, lo
; CHECK-SD-NEXT:    csel x2, x8, x2, lo
; CHECK-SD-NEXT:    subs x8, x4, x10
; CHECK-SD-NEXT:    sbcs x9, x5, x11
; CHECK-SD-NEXT:    ldp x10, x11, [sp]
; CHECK-SD-NEXT:    csel x4, x8, x4, lo
; CHECK-SD-NEXT:    csel x5, x9, x5, lo
; CHECK-SD-NEXT:    subs x8, x0, x10
; CHECK-SD-NEXT:    ldp x9, x10, [sp, #48]
; CHECK-SD-NEXT:    sbcs x11, x1, x11
; CHECK-SD-NEXT:    csel x1, x11, x1, lo
; CHECK-SD-NEXT:    csel x0, x8, x0, lo
; CHECK-SD-NEXT:    subs x8, x6, x9
; CHECK-SD-NEXT:    sbcs x9, x7, x10
; CHECK-SD-NEXT:    csel x6, x8, x6, lo
; CHECK-SD-NEXT:    csel x7, x9, x7, lo
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: usub_v4i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    str x23, [sp, #-48]! // 8-byte Folded Spill
; CHECK-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    .cfi_offset w23, -48
; CHECK-GI-NEXT:    ldp x8, x9, [sp, #48]
; CHECK-GI-NEXT:    ldp x10, x11, [sp, #64]
; CHECK-GI-NEXT:    ldp x12, x13, [sp, #80]
; CHECK-GI-NEXT:    subs x14, x0, x8
; CHECK-GI-NEXT:    ldp x17, x18, [sp, #96]
; CHECK-GI-NEXT:    sbc x15, x1, x9
; CHECK-GI-NEXT:    subs x16, x2, x10
; CHECK-GI-NEXT:    sbc x19, x3, x11
; CHECK-GI-NEXT:    subs x20, x4, x12
; CHECK-GI-NEXT:    sbc x21, x5, x13
; CHECK-GI-NEXT:    subs x22, x6, x17
; CHECK-GI-NEXT:    sbc x23, x7, x18
; CHECK-GI-NEXT:    cmp x0, x8
; CHECK-GI-NEXT:    cset w8, lo
; CHECK-GI-NEXT:    cmp x1, x9
; CHECK-GI-NEXT:    cset w9, lo
; CHECK-GI-NEXT:    csel w8, w8, w9, eq
; CHECK-GI-NEXT:    cmp x2, x10
; CHECK-GI-NEXT:    cset w9, lo
; CHECK-GI-NEXT:    cmp x3, x11
; CHECK-GI-NEXT:    cset w10, lo
; CHECK-GI-NEXT:    csel w9, w9, w10, eq
; CHECK-GI-NEXT:    cmp x4, x12
; CHECK-GI-NEXT:    cset w10, lo
; CHECK-GI-NEXT:    cmp x5, x13
; CHECK-GI-NEXT:    cset w11, lo
; CHECK-GI-NEXT:    csel w10, w10, w11, eq
; CHECK-GI-NEXT:    cmp x6, x17
; CHECK-GI-NEXT:    cset w11, lo
; CHECK-GI-NEXT:    cmp x7, x18
; CHECK-GI-NEXT:    cset w12, lo
; CHECK-GI-NEXT:    csel w11, w11, w12, eq
; CHECK-GI-NEXT:    tst w8, #0x1
; CHECK-GI-NEXT:    csel x0, x14, x0, ne
; CHECK-GI-NEXT:    csel x1, x15, x1, ne
; CHECK-GI-NEXT:    tst w9, #0x1
; CHECK-GI-NEXT:    csel x2, x16, x2, ne
; CHECK-GI-NEXT:    csel x3, x19, x3, ne
; CHECK-GI-NEXT:    tst w10, #0x1
; CHECK-GI-NEXT:    csel x4, x20, x4, ne
; CHECK-GI-NEXT:    csel x5, x21, x5, ne
; CHECK-GI-NEXT:    tst w11, #0x1
; CHECK-GI-NEXT:    csel x6, x22, x6, ne
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x7, x23, x7, ne
; CHECK-GI-NEXT:    ldr x23, [sp], #48 // 8-byte Folded Reload
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i128>, <4 x i1>} @llvm.usub.with.overflow(<4 x i128> %a, <4 x i128> %b)
  %e0 = extractvalue {<4 x i128>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i128>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i128> %e0, <4 x i128> %a
  ret <4 x i128> %r
}

define <4 x i128> @ssub_v4i128(<4 x i128> %a, <4 x i128> %b) {
; CHECK-SD-LABEL: ssub_v4i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    ldp x8, x9, [sp, #16]
; CHECK-SD-NEXT:    ldp x10, x11, [sp, #32]
; CHECK-SD-NEXT:    subs x8, x2, x8
; CHECK-SD-NEXT:    sbcs x9, x3, x9
; CHECK-SD-NEXT:    csel x3, x9, x3, vs
; CHECK-SD-NEXT:    csel x2, x8, x2, vs
; CHECK-SD-NEXT:    subs x8, x4, x10
; CHECK-SD-NEXT:    sbcs x9, x5, x11
; CHECK-SD-NEXT:    ldp x10, x11, [sp]
; CHECK-SD-NEXT:    csel x4, x8, x4, vs
; CHECK-SD-NEXT:    csel x5, x9, x5, vs
; CHECK-SD-NEXT:    subs x8, x0, x10
; CHECK-SD-NEXT:    ldp x9, x10, [sp, #48]
; CHECK-SD-NEXT:    sbcs x11, x1, x11
; CHECK-SD-NEXT:    csel x1, x11, x1, vs
; CHECK-SD-NEXT:    csel x0, x8, x0, vs
; CHECK-SD-NEXT:    subs x8, x6, x9
; CHECK-SD-NEXT:    sbcs x9, x7, x10
; CHECK-SD-NEXT:    csel x6, x8, x6, vs
; CHECK-SD-NEXT:    csel x7, x9, x7, vs
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: ssub_v4i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    str x23, [sp, #-48]! // 8-byte Folded Spill
; CHECK-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    .cfi_offset w23, -48
; CHECK-GI-NEXT:    ldp x16, x17, [sp, #48]
; CHECK-GI-NEXT:    ldp x18, x19, [sp, #64]
; CHECK-GI-NEXT:    ldp x20, x21, [sp, #80]
; CHECK-GI-NEXT:    subs x13, x0, x16
; CHECK-GI-NEXT:    ldp x22, x23, [sp, #96]
; CHECK-GI-NEXT:    sbc x14, x1, x17
; CHECK-GI-NEXT:    subs x10, x2, x18
; CHECK-GI-NEXT:    sbc x15, x3, x19
; CHECK-GI-NEXT:    subs x11, x4, x20
; CHECK-GI-NEXT:    sbc x12, x5, x21
; CHECK-GI-NEXT:    subs x8, x6, x22
; CHECK-GI-NEXT:    sbc x9, x7, x23
; CHECK-GI-NEXT:    cmp x0, x16
; CHECK-GI-NEXT:    cset w16, lo
; CHECK-GI-NEXT:    cmp x1, x17
; CHECK-GI-NEXT:    cset w17, lt
; CHECK-GI-NEXT:    csel w16, w16, w17, eq
; CHECK-GI-NEXT:    cmp x2, x18
; CHECK-GI-NEXT:    cset w17, lo
; CHECK-GI-NEXT:    cmp x3, x19
; CHECK-GI-NEXT:    fmov s0, w16
; CHECK-GI-NEXT:    cset w18, lt
; CHECK-GI-NEXT:    csel w17, w17, w18, eq
; CHECK-GI-NEXT:    cmp x4, x20
; CHECK-GI-NEXT:    cset w18, lo
; CHECK-GI-NEXT:    cmp x5, x21
; CHECK-GI-NEXT:    mov v0.h[1], w17
; CHECK-GI-NEXT:    cset w19, lt
; CHECK-GI-NEXT:    csel w18, w18, w19, eq
; CHECK-GI-NEXT:    cmp x6, x22
; CHECK-GI-NEXT:    cset w19, lo
; CHECK-GI-NEXT:    cmp x7, x23
; CHECK-GI-NEXT:    cset w20, lt
; CHECK-GI-NEXT:    mov v0.h[2], w18
; CHECK-GI-NEXT:    csel w19, w19, w20, eq
; CHECK-GI-NEXT:    cmp x14, #0
; CHECK-GI-NEXT:    cset w20, mi
; CHECK-GI-NEXT:    csel w20, wzr, w20, eq
; CHECK-GI-NEXT:    cmp x15, #0
; CHECK-GI-NEXT:    fmov s1, w20
; CHECK-GI-NEXT:    cset w21, mi
; CHECK-GI-NEXT:    mov v0.h[3], w19
; CHECK-GI-NEXT:    csel w16, wzr, w21, eq
; CHECK-GI-NEXT:    cmp x12, #0
; CHECK-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
; CHECK-GI-NEXT:    mov v1.h[1], w16
; CHECK-GI-NEXT:    cset w16, mi
; CHECK-GI-NEXT:    csel w16, wzr, w16, eq
; CHECK-GI-NEXT:    cmp x9, #0
; CHECK-GI-NEXT:    mov v1.h[2], w16
; CHECK-GI-NEXT:    cset w16, mi
; CHECK-GI-NEXT:    csel w16, wzr, w16, eq
; CHECK-GI-NEXT:    mov v1.h[3], w16
; CHECK-GI-NEXT:    eor v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    umov w16, v0.h[0]
; CHECK-GI-NEXT:    umov w17, v0.h[1]
; CHECK-GI-NEXT:    umov w18, v0.h[2]
; CHECK-GI-NEXT:    umov w19, v0.h[3]
; CHECK-GI-NEXT:    and w16, w16, #0x1
; CHECK-GI-NEXT:    and w17, w17, #0x1
; CHECK-GI-NEXT:    and w18, w18, #0x1
; CHECK-GI-NEXT:    tst w16, #0x1
; CHECK-GI-NEXT:    and w19, w19, #0x1
; CHECK-GI-NEXT:    csel x0, x13, x0, ne
; CHECK-GI-NEXT:    csel x1, x14, x1, ne
; CHECK-GI-NEXT:    tst w17, #0x1
; CHECK-GI-NEXT:    csel x2, x10, x2, ne
; CHECK-GI-NEXT:    csel x3, x15, x3, ne
; CHECK-GI-NEXT:    tst w18, #0x1
; CHECK-GI-NEXT:    csel x4, x11, x4, ne
; CHECK-GI-NEXT:    csel x5, x12, x5, ne
; CHECK-GI-NEXT:    tst w19, #0x1
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x6, x8, x6, ne
; CHECK-GI-NEXT:    csel x7, x9, x7, ne
; CHECK-GI-NEXT:    ldr x23, [sp], #48 // 8-byte Folded Reload
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i128>, <4 x i1>} @llvm.ssub.with.overflow(<4 x i128> %a, <4 x i128> %b)
  %e0 = extractvalue {<4 x i128>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i128>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i128> %e0, <4 x i128> %a
  ret <4 x i128> %r
}

define <4 x i128> @umul_v4i128(<4 x i128> %a, <4 x i128> %b) {
; CHECK-SD-LABEL: umul_v4i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    str x25, [sp, #-64]! // 8-byte Folded Spill
; CHECK-SD-NEXT:    stp x24, x23, [sp, #16] // 16-byte Folded Spill
; CHECK-SD-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
; CHECK-SD-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
; CHECK-SD-NEXT:    .cfi_offset w19, -8
; CHECK-SD-NEXT:    .cfi_offset w20, -16
; CHECK-SD-NEXT:    .cfi_offset w21, -24
; CHECK-SD-NEXT:    .cfi_offset w22, -32
; CHECK-SD-NEXT:    .cfi_offset w23, -40
; CHECK-SD-NEXT:    .cfi_offset w24, -48
; CHECK-SD-NEXT:    .cfi_offset w25, -64
; CHECK-SD-NEXT:    ldp x8, x10, [sp, #112]
; CHECK-SD-NEXT:    cmp x7, #0
; CHECK-SD-NEXT:    ldp x22, x19, [sp, #80]
; CHECK-SD-NEXT:    mul x9, x10, x6
; CHECK-SD-NEXT:    ccmp x10, #0, #4, ne
; CHECK-SD-NEXT:    umulh x12, x7, x8
; CHECK-SD-NEXT:    umulh x11, x10, x6
; CHECK-SD-NEXT:    ldp x25, x10, [sp, #64]
; CHECK-SD-NEXT:    madd x13, x7, x8, x9
; CHECK-SD-NEXT:    ldp x9, x14, [sp, #96]
; CHECK-SD-NEXT:    ccmp xzr, x12, #0, eq
; CHECK-SD-NEXT:    umulh x15, x6, x8
; CHECK-SD-NEXT:    ccmp xzr, x11, #0, eq
; CHECK-SD-NEXT:    mul x17, x14, x4
; CHECK-SD-NEXT:    cset w11, ne
; CHECK-SD-NEXT:    umulh x18, x5, x9
; CHECK-SD-NEXT:    adds x13, x15, x13
; CHECK-SD-NEXT:    umulh x16, x14, x4
; CHECK-SD-NEXT:    csinc w11, w11, wzr, lo
; CHECK-SD-NEXT:    cmp x5, #0
; CHECK-SD-NEXT:    ccmp x14, #0, #4, ne
; CHECK-SD-NEXT:    madd x17, x5, x9, x17
; CHECK-SD-NEXT:    ccmp xzr, x18, #0, eq
; CHECK-SD-NEXT:    umulh x20, x4, x9
; CHECK-SD-NEXT:    ccmp xzr, x16, #0, eq
; CHECK-SD-NEXT:    mul x23, x19, x2
; CHECK-SD-NEXT:    cset w18, ne
; CHECK-SD-NEXT:    umulh x24, x3, x22
; CHECK-SD-NEXT:    adds x17, x20, x17
; CHECK-SD-NEXT:    umulh x21, x19, x2
; CHECK-SD-NEXT:    csinc w18, w18, wzr, lo
; CHECK-SD-NEXT:    cmp x3, #0
; CHECK-SD-NEXT:    ccmp x19, #0, #4, ne
; CHECK-SD-NEXT:    madd x23, x3, x22, x23
; CHECK-SD-NEXT:    ccmp xzr, x24, #0, eq
; CHECK-SD-NEXT:    umulh x12, x2, x22
; CHECK-SD-NEXT:    ccmp xzr, x21, #0, eq
; CHECK-SD-NEXT:    mul x14, x10, x0
; CHECK-SD-NEXT:    cset w20, ne
; CHECK-SD-NEXT:    umulh x16, x1, x25
; CHECK-SD-NEXT:    adds x12, x12, x23
; CHECK-SD-NEXT:    ldp x24, x23, [sp, #16] // 16-byte Folded Reload
; CHECK-SD-NEXT:    umulh x15, x10, x0
; CHECK-SD-NEXT:    csinc w20, w20, wzr, lo
; CHECK-SD-NEXT:    cmp x1, #0
; CHECK-SD-NEXT:    ccmp x10, #0, #4, ne
; CHECK-SD-NEXT:    madd x14, x1, x25, x14
; CHECK-SD-NEXT:    ccmp xzr, x16, #0, eq
; CHECK-SD-NEXT:    umulh x19, x0, x25
; CHECK-SD-NEXT:    ccmp xzr, x15, #0, eq
; CHECK-SD-NEXT:    mul x10, x0, x25
; CHECK-SD-NEXT:    cset w16, ne
; CHECK-SD-NEXT:    mul x15, x2, x22
; CHECK-SD-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
; CHECK-SD-NEXT:    adds x14, x19, x14
; CHECK-SD-NEXT:    mul x9, x4, x9
; CHECK-SD-NEXT:    csinc w16, w16, wzr, lo
; CHECK-SD-NEXT:    cmp w16, #0
; CHECK-SD-NEXT:    mul x8, x6, x8
; CHECK-SD-NEXT:    csel x0, x10, x0, ne
; CHECK-SD-NEXT:    csel x1, x14, x1, ne
; CHECK-SD-NEXT:    cmp w20, #0
; CHECK-SD-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
; CHECK-SD-NEXT:    csel x2, x15, x2, ne
; CHECK-SD-NEXT:    csel x3, x12, x3, ne
; CHECK-SD-NEXT:    cmp w18, #0
; CHECK-SD-NEXT:    csel x4, x9, x4, ne
; CHECK-SD-NEXT:    csel x5, x17, x5, ne
; CHECK-SD-NEXT:    cmp w11, #0
; CHECK-SD-NEXT:    csel x7, x13, x7, ne
; CHECK-SD-NEXT:    csel x6, x8, x6, ne
; CHECK-SD-NEXT:    ldr x25, [sp], #64 // 8-byte Folded Reload
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: umul_v4i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub sp, sp, #208
; CHECK-GI-NEXT:    stp x29, x30, [sp, #112] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x28, x27, [sp, #128] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x26, x25, [sp, #144] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x24, x23, [sp, #160] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x22, x21, [sp, #176] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #192] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 208
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    .cfi_offset w23, -40
; CHECK-GI-NEXT:    .cfi_offset w24, -48
; CHECK-GI-NEXT:    .cfi_offset w25, -56
; CHECK-GI-NEXT:    .cfi_offset w26, -64
; CHECK-GI-NEXT:    .cfi_offset w27, -72
; CHECK-GI-NEXT:    .cfi_offset w28, -80
; CHECK-GI-NEXT:    .cfi_offset w30, -88
; CHECK-GI-NEXT:    .cfi_offset w29, -96
; CHECK-GI-NEXT:    ldp x11, x19, [sp, #208]
; CHECK-GI-NEXT:    mov x22, x2
; CHECK-GI-NEXT:    ldp x23, x25, [sp, #224]
; CHECK-GI-NEXT:    mov x27, x6
; CHECK-GI-NEXT:    mov x21, x0
; CHECK-GI-NEXT:    mov x24, x4
; CHECK-GI-NEXT:    mov x28, x1
; CHECK-GI-NEXT:    umulh x9, x11, xzr
; CHECK-GI-NEXT:    mov x17, x7
; CHECK-GI-NEXT:    umulh x8, x1, x19
; CHECK-GI-NEXT:    umulh x20, x2, x23
; CHECK-GI-NEXT:    mul x6, x2, x25
; CHECK-GI-NEXT:    stp x8, x9, [sp, #96] // 16-byte Folded Spill
; CHECK-GI-NEXT:    mul x9, x3, x25
; CHECK-GI-NEXT:    umulh x8, x3, x25
; CHECK-GI-NEXT:    umulh x18, x2, x25
; CHECK-GI-NEXT:    ldp x2, x30, [sp, #240]
; CHECK-GI-NEXT:    umulh x14, x0, x11
; CHECK-GI-NEXT:    stp x9, x8, [sp, #80] // 16-byte Folded Spill
; CHECK-GI-NEXT:    mul x9, x4, x30
; CHECK-GI-NEXT:    umulh x8, x4, x30
; CHECK-GI-NEXT:    mul x15, x0, x19
; CHECK-GI-NEXT:    umulh x16, x0, x19
; CHECK-GI-NEXT:    stp x8, x9, [sp, #32] // 16-byte Folded Spill
; CHECK-GI-NEXT:    mul x8, x21, x11
; CHECK-GI-NEXT:    mul x0, x1, x19
; CHECK-GI-NEXT:    madd x19, x21, x19, x14
; CHECK-GI-NEXT:    adds x14, x14, x15
; CHECK-GI-NEXT:    stp x3, x8, [sp, #48] // 16-byte Folded Spill
; CHECK-GI-NEXT:    cset w15, hs
; CHECK-GI-NEXT:    madd x29, x22, x25, x20
; CHECK-GI-NEXT:    and x15, x15, #0x1
; CHECK-GI-NEXT:    madd x9, x1, x11, x19
; CHECK-GI-NEXT:    madd x8, x3, x23, x29
; CHECK-GI-NEXT:    mul x12, x1, x11
; CHECK-GI-NEXT:    umulh x26, x4, x2
; CHECK-GI-NEXT:    stp x8, x9, [sp, #64] // 16-byte Folded Spill
; CHECK-GI-NEXT:    umulh x8, x5, x30
; CHECK-GI-NEXT:    cmn x14, x12
; CHECK-GI-NEXT:    umulh x13, x1, x11
; CHECK-GI-NEXT:    mul x4, x5, x30
; CHECK-GI-NEXT:    str x8, [sp, #8] // 8-byte Spill
; CHECK-GI-NEXT:    madd x8, x24, x30, x26
; CHECK-GI-NEXT:    mul x9, x3, x23
; CHECK-GI-NEXT:    umulh x10, x3, x23
; CHECK-GI-NEXT:    umulh x30, x23, xzr
; CHECK-GI-NEXT:    mul x11, x22, x23
; CHECK-GI-NEXT:    cset w23, hs
; CHECK-GI-NEXT:    adds x16, x13, x16
; CHECK-GI-NEXT:    and x23, x23, #0x1
; CHECK-GI-NEXT:    madd x8, x5, x2, x8
; CHECK-GI-NEXT:    add x15, x15, x23
; CHECK-GI-NEXT:    cset w23, hs
; CHECK-GI-NEXT:    adds x16, x16, x0
; CHECK-GI-NEXT:    cset w0, hs
; CHECK-GI-NEXT:    adds x29, x16, x15
; CHECK-GI-NEXT:    umulh x12, x5, x2
; CHECK-GI-NEXT:    cset w1, hs
; CHECK-GI-NEXT:    adds x16, x20, x6
; CHECK-GI-NEXT:    cset w20, hs
; CHECK-GI-NEXT:    cmn x16, x9
; CHECK-GI-NEXT:    umulh x3, x21, xzr
; CHECK-GI-NEXT:    stp x11, x8, [sp, #16] // 16-byte Folded Spill
; CHECK-GI-NEXT:    cset w9, hs
; CHECK-GI-NEXT:    ldr x8, [sp, #80] // 8-byte Reload
; CHECK-GI-NEXT:    adds x10, x10, x18
; CHECK-GI-NEXT:    and x18, x20, #0x1
; CHECK-GI-NEXT:    and x9, x9, #0x1
; CHECK-GI-NEXT:    mul x11, x5, x2
; CHECK-GI-NEXT:    cset w15, hs
; CHECK-GI-NEXT:    adds x10, x10, x8
; CHECK-GI-NEXT:    add x9, x18, x9
; CHECK-GI-NEXT:    cset w20, hs
; CHECK-GI-NEXT:    and x1, x1, #0x1
; CHECK-GI-NEXT:    adds x25, x10, x9
; CHECK-GI-NEXT:    ldp x10, x8, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT:    ldp x16, x18, [sp, #256]
; CHECK-GI-NEXT:    cset w13, hs
; CHECK-GI-NEXT:    umulh x7, x22, xzr
; CHECK-GI-NEXT:    and x15, x15, #0x1
; CHECK-GI-NEXT:    and x20, x20, #0x1
; CHECK-GI-NEXT:    adds x26, x26, x8
; CHECK-GI-NEXT:    add x15, x15, x20
; CHECK-GI-NEXT:    and x13, x13, #0x1
; CHECK-GI-NEXT:    umulh x14, x27, x16
; CHECK-GI-NEXT:    cset w8, hs
; CHECK-GI-NEXT:    cmn x26, x11
; CHECK-GI-NEXT:    cset w26, hs
; CHECK-GI-NEXT:    and x8, x8, #0x1
; CHECK-GI-NEXT:    adds x12, x12, x10
; CHECK-GI-NEXT:    mul x11, x27, x18
; CHECK-GI-NEXT:    and x26, x26, #0x1
; CHECK-GI-NEXT:    add x8, x8, x26
; CHECK-GI-NEXT:    cset w26, hs
; CHECK-GI-NEXT:    adds x12, x12, x4
; CHECK-GI-NEXT:    mul x9, x17, x16
; CHECK-GI-NEXT:    cset w10, hs
; CHECK-GI-NEXT:    adds x19, x12, x8
; CHECK-GI-NEXT:    cset w12, hs
; CHECK-GI-NEXT:    and x10, x10, #0x1
; CHECK-GI-NEXT:    umulh x4, x2, xzr
; CHECK-GI-NEXT:    and x12, x12, #0x1
; CHECK-GI-NEXT:    adds x11, x14, x11
; CHECK-GI-NEXT:    cset w8, hs
; CHECK-GI-NEXT:    umulh x6, x24, xzr
; CHECK-GI-NEXT:    cmn x11, x9
; CHECK-GI-NEXT:    and x9, x23, #0x1
; CHECK-GI-NEXT:    and x11, x0, #0x1
; CHECK-GI-NEXT:    add x9, x9, x11
; CHECK-GI-NEXT:    ldp x23, x11, [sp, #96] // 16-byte Folded Reload
; CHECK-GI-NEXT:    umulh x0, x17, x16
; CHECK-GI-NEXT:    add x9, x9, x1
; CHECK-GI-NEXT:    ldr x1, [sp, #88] // 8-byte Reload
; CHECK-GI-NEXT:    and x8, x8, #0x1
; CHECK-GI-NEXT:    add x11, x11, x23
; CHECK-GI-NEXT:    umulh x23, x27, x18
; CHECK-GI-NEXT:    add x1, x30, x1
; CHECK-GI-NEXT:    add x11, x11, x3
; CHECK-GI-NEXT:    mul x3, x17, x18
; CHECK-GI-NEXT:    add x9, x11, x9
; CHECK-GI-NEXT:    add x11, x15, x13
; CHECK-GI-NEXT:    add x13, x1, x7
; CHECK-GI-NEXT:    and x1, x26, #0x1
; CHECK-GI-NEXT:    orr x9, x29, x9
; CHECK-GI-NEXT:    add x11, x13, x11
; CHECK-GI-NEXT:    cset w13, hs
; CHECK-GI-NEXT:    add x10, x1, x10
; CHECK-GI-NEXT:    adds x0, x0, x23
; CHECK-GI-NEXT:    and x13, x13, #0x1
; CHECK-GI-NEXT:    madd x14, x27, x18, x14
; CHECK-GI-NEXT:    add x8, x8, x13
; CHECK-GI-NEXT:    cset w13, hs
; CHECK-GI-NEXT:    add x10, x10, x12
; CHECK-GI-NEXT:    adds x0, x0, x3
; CHECK-GI-NEXT:    umulh x15, x16, xzr
; CHECK-GI-NEXT:    and x13, x13, #0x1
; CHECK-GI-NEXT:    cset w12, hs
; CHECK-GI-NEXT:    adds x8, x0, x8
; CHECK-GI-NEXT:    ldr x0, [sp, #8] // 8-byte Reload
; CHECK-GI-NEXT:    umulh x7, x17, x18
; CHECK-GI-NEXT:    and x12, x12, #0x1
; CHECK-GI-NEXT:    orr x11, x25, x11
; CHECK-GI-NEXT:    add x12, x13, x12
; CHECK-GI-NEXT:    cset w13, hs
; CHECK-GI-NEXT:    cmp x9, #0
; CHECK-GI-NEXT:    umulh x1, x27, xzr
; CHECK-GI-NEXT:    add x0, x4, x0
; CHECK-GI-NEXT:    and x13, x13, #0x1
; CHECK-GI-NEXT:    add x18, x0, x6
; CHECK-GI-NEXT:    add x12, x12, x13
; CHECK-GI-NEXT:    madd x9, x17, x16, x14
; CHECK-GI-NEXT:    ldr x14, [sp, #56] // 8-byte Reload
; CHECK-GI-NEXT:    add x10, x18, x10
; CHECK-GI-NEXT:    add x15, x15, x7
; CHECK-GI-NEXT:    orr x10, x19, x10
; CHECK-GI-NEXT:    csel x0, x14, x21, ne
; CHECK-GI-NEXT:    ldr x14, [sp, #72] // 8-byte Reload
; CHECK-GI-NEXT:    mul x20, x24, x2
; CHECK-GI-NEXT:    add x13, x15, x1
; CHECK-GI-NEXT:    ldp x26, x25, [sp, #144] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x1, x14, x28, ne
; CHECK-GI-NEXT:    cmp x11, #0
; CHECK-GI-NEXT:    ldr x11, [sp, #16] // 8-byte Reload
; CHECK-GI-NEXT:    add x12, x13, x12
; CHECK-GI-NEXT:    mul x13, x27, x16
; CHECK-GI-NEXT:    ldr x14, [sp, #48] // 8-byte Reload
; CHECK-GI-NEXT:    csel x2, x11, x22, ne
; CHECK-GI-NEXT:    ldr x11, [sp, #64] // 8-byte Reload
; CHECK-GI-NEXT:    orr x8, x8, x12
; CHECK-GI-NEXT:    ldp x22, x21, [sp, #176] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x3, x11, x14, ne
; CHECK-GI-NEXT:    cmp x10, #0
; CHECK-GI-NEXT:    ldr x10, [sp, #24] // 8-byte Reload
; CHECK-GI-NEXT:    csel x4, x20, x24, ne
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #192] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x5, x10, x5, ne
; CHECK-GI-NEXT:    cmp x8, #0
; CHECK-GI-NEXT:    csel x6, x13, x27, ne
; CHECK-GI-NEXT:    ldp x24, x23, [sp, #160] // 16-byte Folded Reload
; CHECK-GI-NEXT:    ldp x28, x27, [sp, #128] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x7, x9, x17, ne
; CHECK-GI-NEXT:    ldp x29, x30, [sp, #112] // 16-byte Folded Reload
; CHECK-GI-NEXT:    add sp, sp, #208
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i128>, <4 x i1>} @llvm.umul.with.overflow(<4 x i128> %a, <4 x i128> %b)
  %e0 = extractvalue {<4 x i128>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i128>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i128> %e0, <4 x i128> %a
  ret <4 x i128> %r
}

define <4 x i128> @smul_v4i128(<4 x i128> %a, <4 x i128> %b) {
; CHECK-SD-LABEL: smul_v4i128:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    sub sp, sp, #160
; CHECK-SD-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
; CHECK-SD-NEXT:    stp x28, x27, [sp, #80] // 16-byte Folded Spill
; CHECK-SD-NEXT:    stp x26, x25, [sp, #96] // 16-byte Folded Spill
; CHECK-SD-NEXT:    stp x24, x23, [sp, #112] // 16-byte Folded Spill
; CHECK-SD-NEXT:    stp x22, x21, [sp, #128] // 16-byte Folded Spill
; CHECK-SD-NEXT:    stp x20, x19, [sp, #144] // 16-byte Folded Spill
; CHECK-SD-NEXT:    .cfi_def_cfa_offset 160
; CHECK-SD-NEXT:    .cfi_offset w19, -8
; CHECK-SD-NEXT:    .cfi_offset w20, -16
; CHECK-SD-NEXT:    .cfi_offset w21, -24
; CHECK-SD-NEXT:    .cfi_offset w22, -32
; CHECK-SD-NEXT:    .cfi_offset w23, -40
; CHECK-SD-NEXT:    .cfi_offset w24, -48
; CHECK-SD-NEXT:    .cfi_offset w25, -56
; CHECK-SD-NEXT:    .cfi_offset w26, -64
; CHECK-SD-NEXT:    .cfi_offset w27, -72
; CHECK-SD-NEXT:    .cfi_offset w28, -80
; CHECK-SD-NEXT:    .cfi_offset w30, -88
; CHECK-SD-NEXT:    .cfi_offset w29, -96
; CHECK-SD-NEXT:    mov x19, x7
; CHECK-SD-NEXT:    ldp x23, x20, [sp, #200]
; CHECK-SD-NEXT:    asr x9, x19, #63
; CHECK-SD-NEXT:    mov x7, x6
; CHECK-SD-NEXT:    mov x6, x5
; CHECK-SD-NEXT:    ldp x26, x17, [sp, #184]
; CHECK-SD-NEXT:    mov x5, x4
; CHECK-SD-NEXT:    mul x21, x9, x20
; CHECK-SD-NEXT:    ldr x9, [sp, #216]
; CHECK-SD-NEXT:    mov x4, x3
; CHECK-SD-NEXT:    mov x3, x2
; CHECK-SD-NEXT:    mov x2, x1
; CHECK-SD-NEXT:    mov x1, x0
; CHECK-SD-NEXT:    mul x8, x19, x9
; CHECK-SD-NEXT:    asr x10, x9, #63
; CHECK-SD-NEXT:    asr x25, x4, #63
; CHECK-SD-NEXT:    ldr x14, [sp, #176]
; CHECK-SD-NEXT:    asr x28, x26, #63
; CHECK-SD-NEXT:    mul x15, x7, x10
; CHECK-SD-NEXT:    asr x10, x6, #63
; CHECK-SD-NEXT:    umulh x13, x7, x9
; CHECK-SD-NEXT:    str x8, [sp, #48] // 8-byte Spill
; CHECK-SD-NEXT:    mul x8, x10, x17
; CHECK-SD-NEXT:    asr x10, x23, #63
; CHECK-SD-NEXT:    mul x18, x7, x9
; CHECK-SD-NEXT:    smulh x30, x19, x9
; CHECK-SD-NEXT:    str x8, [sp, #56] // 8-byte Spill
; CHECK-SD-NEXT:    umulh x9, x5, x23
; CHECK-SD-NEXT:    smulh x8, x6, x23
; CHECK-SD-NEXT:    umulh x11, x7, x20
; CHECK-SD-NEXT:    umulh x0, x19, x20
; CHECK-SD-NEXT:    stp x9, x8, [sp, #32] // 16-byte Folded Spill
; CHECK-SD-NEXT:    mul x9, x6, x23
; CHECK-SD-NEXT:    umulh x8, x3, x26
; CHECK-SD-NEXT:    mul x27, x25, x14
; CHECK-SD-NEXT:    umulh x12, x5, x17
; CHECK-SD-NEXT:    stp x8, x9, [sp, #16] // 16-byte Folded Spill
; CHECK-SD-NEXT:    smulh x8, x4, x26
; CHECK-SD-NEXT:    mul x22, x5, x10
; CHECK-SD-NEXT:    umulh x16, x6, x17
; CHECK-SD-NEXT:    str x8, [sp, #8] // 8-byte Spill
; CHECK-SD-NEXT:    mul x8, x19, x20
; CHECK-SD-NEXT:    mul x24, x5, x23
; CHECK-SD-NEXT:    mul x29, x3, x26
; CHECK-SD-NEXT:    adds x9, x8, x11
; CHECK-SD-NEXT:    adc x8, x0, x21
; CHECK-SD-NEXT:    adds x25, x18, x9
; CHECK-SD-NEXT:    mul x11, x6, x17
; CHECK-SD-NEXT:    adc x9, x13, x15
; CHECK-SD-NEXT:    asr x13, x8, #63
; CHECK-SD-NEXT:    asr x10, x9, #63
; CHECK-SD-NEXT:    adds x8, x8, x9
; CHECK-SD-NEXT:    ldr x9, [sp, #48] // 8-byte Reload
; CHECK-SD-NEXT:    umulh x18, x3, x14
; CHECK-SD-NEXT:    adc x10, x13, x10
; CHECK-SD-NEXT:    adds x8, x9, x8
; CHECK-SD-NEXT:    ldr x13, [sp, #160]
; CHECK-SD-NEXT:    str x8, [sp, #48] // 8-byte Spill
; CHECK-SD-NEXT:    ldr x8, [sp, #56] // 8-byte Reload
; CHECK-SD-NEXT:    adc x23, x30, x10
; CHECK-SD-NEXT:    adds x10, x11, x12
; CHECK-SD-NEXT:    mul x15, x4, x14
; CHECK-SD-NEXT:    asr x11, x2, #63
; CHECK-SD-NEXT:    adc x16, x16, x8
; CHECK-SD-NEXT:    ldp x9, x8, [sp, #24] // 16-byte Folded Reload
; CHECK-SD-NEXT:    adds x21, x24, x10
; CHECK-SD-NEXT:    umulh x0, x4, x14
; CHECK-SD-NEXT:    asr x24, x16, #63
; CHECK-SD-NEXT:    ldr x10, [sp, #40] // 8-byte Reload
; CHECK-SD-NEXT:    adc x22, x8, x22
; CHECK-SD-NEXT:    mul x28, x3, x28
; CHECK-SD-NEXT:    asr x8, x22, #63
; CHECK-SD-NEXT:    adds x16, x16, x22
; CHECK-SD-NEXT:    mul x26, x4, x26
; CHECK-SD-NEXT:    adc x8, x24, x8
; CHECK-SD-NEXT:    adds x30, x9, x16
; CHECK-SD-NEXT:    ldr x24, [sp, #168]
; CHECK-SD-NEXT:    adc x16, x10, x8
; CHECK-SD-NEXT:    adds x8, x15, x18
; CHECK-SD-NEXT:    umulh x22, x1, x13
; CHECK-SD-NEXT:    adc x15, x0, x27
; CHECK-SD-NEXT:    adds x10, x29, x8
; CHECK-SD-NEXT:    ldr x8, [sp, #16] // 8-byte Reload
; CHECK-SD-NEXT:    mul x9, x2, x13
; CHECK-SD-NEXT:    asr x0, x15, #63
; CHECK-SD-NEXT:    asr x29, x24, #63
; CHECK-SD-NEXT:    adc x27, x8, x28
; CHECK-SD-NEXT:    umulh x12, x2, x13
; CHECK-SD-NEXT:    asr x28, x27, #63
; CHECK-SD-NEXT:    adds x15, x15, x27
; CHECK-SD-NEXT:    ldr x27, [sp, #8] // 8-byte Reload
; CHECK-SD-NEXT:    mul x11, x11, x13
; CHECK-SD-NEXT:    adc x0, x0, x28
; CHECK-SD-NEXT:    adds x15, x26, x15
; CHECK-SD-NEXT:    adc x0, x27, x0
; CHECK-SD-NEXT:    asr x27, x25, #63
; CHECK-SD-NEXT:    adds x9, x9, x22
; CHECK-SD-NEXT:    mul x8, x1, x24
; CHECK-SD-NEXT:    umulh x18, x1, x24
; CHECK-SD-NEXT:    adc x11, x12, x11
; CHECK-SD-NEXT:    eor x12, x23, x27
; CHECK-SD-NEXT:    ldr x23, [sp, #48] // 8-byte Reload
; CHECK-SD-NEXT:    mul x26, x1, x29
; CHECK-SD-NEXT:    adds x8, x8, x9
; CHECK-SD-NEXT:    eor x23, x23, x27
; CHECK-SD-NEXT:    mul x9, x2, x24
; CHECK-SD-NEXT:    orr x12, x23, x12
; CHECK-SD-NEXT:    asr x23, x10, #63
; CHECK-SD-NEXT:    smulh x22, x2, x24
; CHECK-SD-NEXT:    asr x24, x21, #63
; CHECK-SD-NEXT:    eor x0, x0, x23
; CHECK-SD-NEXT:    adc x18, x18, x26
; CHECK-SD-NEXT:    asr x26, x11, #63
; CHECK-SD-NEXT:    asr x27, x18, #63
; CHECK-SD-NEXT:    adds x11, x11, x18
; CHECK-SD-NEXT:    mul x13, x1, x13
; CHECK-SD-NEXT:    eor x16, x16, x24
; CHECK-SD-NEXT:    eor x24, x30, x24
; CHECK-SD-NEXT:    adc x18, x26, x27
; CHECK-SD-NEXT:    adds x9, x9, x11
; CHECK-SD-NEXT:    asr x11, x8, #63
; CHECK-SD-NEXT:    adc x18, x22, x18
; CHECK-SD-NEXT:    mul x14, x3, x14
; CHECK-SD-NEXT:    orr x16, x24, x16
; CHECK-SD-NEXT:    eor x18, x18, x11
; CHECK-SD-NEXT:    eor x9, x9, x11
; CHECK-SD-NEXT:    eor x11, x15, x23
; CHECK-SD-NEXT:    mul x17, x5, x17
; CHECK-SD-NEXT:    orr x9, x9, x18
; CHECK-SD-NEXT:    orr x11, x11, x0
; CHECK-SD-NEXT:    cmp x9, #0
; CHECK-SD-NEXT:    ldp x24, x23, [sp, #112] // 16-byte Folded Reload
; CHECK-SD-NEXT:    mul x15, x7, x20
; CHECK-SD-NEXT:    csel x0, x13, x1, ne
; CHECK-SD-NEXT:    csel x1, x8, x2, ne
; CHECK-SD-NEXT:    cmp x11, #0
; CHECK-SD-NEXT:    ldp x28, x27, [sp, #80] // 16-byte Folded Reload
; CHECK-SD-NEXT:    csel x2, x14, x3, ne
; CHECK-SD-NEXT:    csel x3, x10, x4, ne
; CHECK-SD-NEXT:    cmp x16, #0
; CHECK-SD-NEXT:    csel x4, x17, x5, ne
; CHECK-SD-NEXT:    csel x5, x21, x6, ne
; CHECK-SD-NEXT:    cmp x12, #0
; CHECK-SD-NEXT:    ldp x22, x21, [sp, #128] // 16-byte Folded Reload
; CHECK-SD-NEXT:    csel x6, x15, x7, ne
; CHECK-SD-NEXT:    csel x7, x25, x19, ne
; CHECK-SD-NEXT:    ldp x20, x19, [sp, #144] // 16-byte Folded Reload
; CHECK-SD-NEXT:    ldp x26, x25, [sp, #96] // 16-byte Folded Reload
; CHECK-SD-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; CHECK-SD-NEXT:    add sp, sp, #160
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: smul_v4i128:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    sub sp, sp, #224
; CHECK-GI-NEXT:    stp x29, x30, [sp, #128] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x28, x27, [sp, #144] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x26, x25, [sp, #160] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x24, x23, [sp, #176] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x22, x21, [sp, #192] // 16-byte Folded Spill
; CHECK-GI-NEXT:    stp x20, x19, [sp, #208] // 16-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 224
; CHECK-GI-NEXT:    .cfi_offset w19, -8
; CHECK-GI-NEXT:    .cfi_offset w20, -16
; CHECK-GI-NEXT:    .cfi_offset w21, -24
; CHECK-GI-NEXT:    .cfi_offset w22, -32
; CHECK-GI-NEXT:    .cfi_offset w23, -40
; CHECK-GI-NEXT:    .cfi_offset w24, -48
; CHECK-GI-NEXT:    .cfi_offset w25, -56
; CHECK-GI-NEXT:    .cfi_offset w26, -64
; CHECK-GI-NEXT:    .cfi_offset w27, -72
; CHECK-GI-NEXT:    .cfi_offset w28, -80
; CHECK-GI-NEXT:    .cfi_offset w30, -88
; CHECK-GI-NEXT:    .cfi_offset w29, -96
; CHECK-GI-NEXT:    ldp x25, x8, [sp, #224]
; CHECK-GI-NEXT:    asr x10, x1, #63
; CHECK-GI-NEXT:    mov x17, x5
; CHECK-GI-NEXT:    ldp x5, x19, [sp, #240]
; CHECK-GI-NEXT:    mov x18, x1
; CHECK-GI-NEXT:    asr x26, x17, #63
; CHECK-GI-NEXT:    mov x11, x6
; CHECK-GI-NEXT:    asr x12, x8, #63
; CHECK-GI-NEXT:    umulh x9, x10, x25
; CHECK-GI-NEXT:    asr x6, x7, #63
; CHECK-GI-NEXT:    asr x16, x19, #63
; CHECK-GI-NEXT:    str x2, [sp, #72] // 8-byte Spill
; CHECK-GI-NEXT:    umulh x13, x1, x8
; CHECK-GI-NEXT:    stp x18, x3, [sp, #8] // 16-byte Folded Spill
; CHECK-GI-NEXT:    str x0, [sp, #48] // 8-byte Spill
; CHECK-GI-NEXT:    umulh x14, x0, x12
; CHECK-GI-NEXT:    umulh x20, x3, x19
; CHECK-GI-NEXT:    add x9, x9, x13
; CHECK-GI-NEXT:    asr x13, x3, #63
; CHECK-GI-NEXT:    umulh x22, x2, x16
; CHECK-GI-NEXT:    add x9, x9, x14
; CHECK-GI-NEXT:    madd x9, x0, x12, x9
; CHECK-GI-NEXT:    mul x14, x0, x12
; CHECK-GI-NEXT:    madd x9, x1, x12, x9
; CHECK-GI-NEXT:    mul x12, x10, x25
; CHECK-GI-NEXT:    umulh x15, x13, x5
; CHECK-GI-NEXT:    umulh x21, x0, x25
; CHECK-GI-NEXT:    str x12, [sp, #112] // 8-byte Spill
; CHECK-GI-NEXT:    madd x12, x10, x8, x9
; CHECK-GI-NEXT:    madd x9, x10, x25, x12
; CHECK-GI-NEXT:    add x10, x15, x20
; CHECK-GI-NEXT:    add x10, x10, x22
; CHECK-GI-NEXT:    ldp x1, x22, [sp, #256]
; CHECK-GI-NEXT:    madd x15, x2, x16, x10
; CHECK-GI-NEXT:    asr x23, x22, #63
; CHECK-GI-NEXT:    umulh x20, x26, x1
; CHECK-GI-NEXT:    str x9, [sp, #120] // 8-byte Spill
; CHECK-GI-NEXT:    madd x15, x3, x16, x15
; CHECK-GI-NEXT:    mul x9, x2, x16
; CHECK-GI-NEXT:    umulh x24, x17, x22
; CHECK-GI-NEXT:    umulh x27, x4, x23
; CHECK-GI-NEXT:    stp x9, x14, [sp, #80] // 16-byte Folded Spill
; CHECK-GI-NEXT:    madd x15, x13, x19, x15
; CHECK-GI-NEXT:    add x16, x20, x24
; CHECK-GI-NEXT:    mov x24, x17
; CHECK-GI-NEXT:    mov x20, x4
; CHECK-GI-NEXT:    mul x10, x13, x5
; CHECK-GI-NEXT:    add x16, x16, x27
; CHECK-GI-NEXT:    madd x9, x13, x5, x15
; CHECK-GI-NEXT:    madd x16, x4, x23, x16
; CHECK-GI-NEXT:    mul x15, x2, x19
; CHECK-GI-NEXT:    str x9, [sp, #104] // 8-byte Spill
; CHECK-GI-NEXT:    madd x16, x17, x23, x16
; CHECK-GI-NEXT:    mul x9, x4, x23
; CHECK-GI-NEXT:    ldp x14, x23, [sp, #272]
; CHECK-GI-NEXT:    madd x13, x26, x22, x16
; CHECK-GI-NEXT:    ldr x16, [sp, #88] // 8-byte Reload
; CHECK-GI-NEXT:    asr x29, x23, #63
; CHECK-GI-NEXT:    umulh x27, x6, x14
; CHECK-GI-NEXT:    stp x10, x9, [sp, #56] // 16-byte Folded Spill
; CHECK-GI-NEXT:    umulh x28, x7, x23
; CHECK-GI-NEXT:    umulh x30, x11, x29
; CHECK-GI-NEXT:    madd x9, x26, x1, x13
; CHECK-GI-NEXT:    add x27, x27, x28
; CHECK-GI-NEXT:    mul x10, x0, x8
; CHECK-GI-NEXT:    add x27, x27, x30
; CHECK-GI-NEXT:    madd x27, x11, x29, x27
; CHECK-GI-NEXT:    str x9, [sp, #96] // 8-byte Spill
; CHECK-GI-NEXT:    umulh x13, x0, x8
; CHECK-GI-NEXT:    adds x10, x21, x10
; CHECK-GI-NEXT:    madd x30, x7, x29, x27
; CHECK-GI-NEXT:    mul x9, x11, x29
; CHECK-GI-NEXT:    mul x29, x18, x8
; CHECK-GI-NEXT:    madd x8, x0, x8, x21
; CHECK-GI-NEXT:    mul x28, x26, x1
; CHECK-GI-NEXT:    mov x26, x7
; CHECK-GI-NEXT:    mov x7, x11
; CHECK-GI-NEXT:    umulh x11, x2, x5
; CHECK-GI-NEXT:    stp x8, x9, [sp, #24] // 16-byte Folded Spill
; CHECK-GI-NEXT:    mov x8, x2
; CHECK-GI-NEXT:    cset w9, hs
; CHECK-GI-NEXT:    umulh x17, x2, x19
; CHECK-GI-NEXT:    and x9, x9, #0x1
; CHECK-GI-NEXT:    madd x8, x8, x19, x11
; CHECK-GI-NEXT:    mul x2, x3, x19
; CHECK-GI-NEXT:    mul x19, x18, x25
; CHECK-GI-NEXT:    str x8, [sp, #40] // 8-byte Spill
; CHECK-GI-NEXT:    umulh x8, x18, x25
; CHECK-GI-NEXT:    ldr x18, [sp, #112] // 8-byte Reload
; CHECK-GI-NEXT:    mul x12, x3, x5
; CHECK-GI-NEXT:    cmn x10, x19
; CHECK-GI-NEXT:    cset w10, hs
; CHECK-GI-NEXT:    umulh x21, x3, x5
; CHECK-GI-NEXT:    adds x8, x8, x13
; CHECK-GI-NEXT:    and x10, x10, #0x1
; CHECK-GI-NEXT:    cset w19, hs
; CHECK-GI-NEXT:    adds x8, x8, x16
; CHECK-GI-NEXT:    add x9, x9, x10
; CHECK-GI-NEXT:    cset w10, hs
; CHECK-GI-NEXT:    adds x8, x8, x29
; CHECK-GI-NEXT:    umulh x13, x4, x1
; CHECK-GI-NEXT:    cset w0, hs
; CHECK-GI-NEXT:    adds x8, x8, x18
; CHECK-GI-NEXT:    and x10, x10, #0x1
; CHECK-GI-NEXT:    mul x16, x4, x22
; CHECK-GI-NEXT:    and x0, x0, #0x1
; CHECK-GI-NEXT:    umulh x18, x4, x22
; CHECK-GI-NEXT:    cset w4, hs
; CHECK-GI-NEXT:    adds x8, x8, x9
; CHECK-GI-NEXT:    str x8, [sp, #112] // 8-byte Spill
; CHECK-GI-NEXT:    cset w29, hs
; CHECK-GI-NEXT:    adds x8, x11, x15
; CHECK-GI-NEXT:    cset w9, hs
; CHECK-GI-NEXT:    cmn x8, x12
; CHECK-GI-NEXT:    ldr x11, [sp, #80] // 8-byte Reload
; CHECK-GI-NEXT:    cset w8, hs
; CHECK-GI-NEXT:    adds x12, x21, x17
; CHECK-GI-NEXT:    and x9, x9, #0x1
; CHECK-GI-NEXT:    cset w17, hs
; CHECK-GI-NEXT:    adds x12, x12, x11
; CHECK-GI-NEXT:    and x8, x8, #0x1
; CHECK-GI-NEXT:    mul x21, x24, x1
; CHECK-GI-NEXT:    ldr x11, [sp, #56] // 8-byte Reload
; CHECK-GI-NEXT:    add x8, x9, x8
; CHECK-GI-NEXT:    cset w9, hs
; CHECK-GI-NEXT:    adds x12, x12, x2
; CHECK-GI-NEXT:    and x17, x17, #0x1
; CHECK-GI-NEXT:    mul x15, x24, x22
; CHECK-GI-NEXT:    cset w2, hs
; CHECK-GI-NEXT:    adds x12, x12, x11
; CHECK-GI-NEXT:    cset w3, hs
; CHECK-GI-NEXT:    adds x8, x12, x8
; CHECK-GI-NEXT:    and x9, x9, #0x1
; CHECK-GI-NEXT:    madd x27, x20, x22, x13
; CHECK-GI-NEXT:    cset w12, hs
; CHECK-GI-NEXT:    adds x13, x13, x16
; CHECK-GI-NEXT:    str x8, [sp, #88] // 8-byte Spill
; CHECK-GI-NEXT:    ldr x8, [sp, #64] // 8-byte Reload
; CHECK-GI-NEXT:    add x9, x17, x9
; CHECK-GI-NEXT:    umulh x22, x24, x1
; CHECK-GI-NEXT:    and x4, x4, #0x1
; CHECK-GI-NEXT:    and x2, x2, #0x1
; CHECK-GI-NEXT:    add x9, x9, x2
; CHECK-GI-NEXT:    and x2, x3, #0x1
; CHECK-GI-NEXT:    madd x16, x6, x23, x30
; CHECK-GI-NEXT:    cset w30, hs
; CHECK-GI-NEXT:    cmn x13, x21
; CHECK-GI-NEXT:    and x13, x19, #0x1
; CHECK-GI-NEXT:    mov x19, x26
; CHECK-GI-NEXT:    add x9, x9, x2
; CHECK-GI-NEXT:    umulh x21, x7, x14
; CHECK-GI-NEXT:    add x10, x13, x10
; CHECK-GI-NEXT:    cset w13, hs
; CHECK-GI-NEXT:    adds x18, x22, x18
; CHECK-GI-NEXT:    add x11, x10, x0
; CHECK-GI-NEXT:    and x13, x13, #0x1
; CHECK-GI-NEXT:    mul x22, x7, x23
; CHECK-GI-NEXT:    cset w0, hs
; CHECK-GI-NEXT:    adds x18, x18, x8
; CHECK-GI-NEXT:    cset w17, hs
; CHECK-GI-NEXT:    adds x15, x18, x15
; CHECK-GI-NEXT:    and x18, x30, #0x1
; CHECK-GI-NEXT:    mul x10, x26, x14
; CHECK-GI-NEXT:    add x13, x18, x13
; CHECK-GI-NEXT:    cset w18, hs
; CHECK-GI-NEXT:    adds x15, x15, x28
; CHECK-GI-NEXT:    and x17, x17, #0x1
; CHECK-GI-NEXT:    ldr x2, [sp, #32] // 8-byte Reload
; CHECK-GI-NEXT:    umulh x30, x26, x14
; CHECK-GI-NEXT:    cset w28, hs
; CHECK-GI-NEXT:    adds x26, x15, x13
; CHECK-GI-NEXT:    cset w15, hs
; CHECK-GI-NEXT:    and x18, x18, #0x1
; CHECK-GI-NEXT:    umulh x8, x7, x23
; CHECK-GI-NEXT:    and x15, x15, #0x1
; CHECK-GI-NEXT:    madd x13, x7, x23, x21
; CHECK-GI-NEXT:    adds x21, x21, x22
; CHECK-GI-NEXT:    cset w22, hs
; CHECK-GI-NEXT:    cmn x21, x10
; CHECK-GI-NEXT:    add x10, x11, x4
; CHECK-GI-NEXT:    and x11, x29, #0x1
; CHECK-GI-NEXT:    mul x3, x19, x23
; CHECK-GI-NEXT:    ldr x23, [sp, #8] // 8-byte Reload
; CHECK-GI-NEXT:    add x10, x10, x11
; CHECK-GI-NEXT:    and x11, x12, #0x1
; CHECK-GI-NEXT:    and x12, x0, #0x1
; CHECK-GI-NEXT:    ldr x0, [sp, #120] // 8-byte Reload
; CHECK-GI-NEXT:    add x9, x9, x11
; CHECK-GI-NEXT:    add x11, x12, x17
; CHECK-GI-NEXT:    cset w12, hs
; CHECK-GI-NEXT:    adds x8, x30, x8
; CHECK-GI-NEXT:    mul x17, x6, x14
; CHECK-GI-NEXT:    add x10, x0, x10
; CHECK-GI-NEXT:    cset w0, hs
; CHECK-GI-NEXT:    adds x8, x8, x2
; CHECK-GI-NEXT:    cset w2, hs
; CHECK-GI-NEXT:    adds x8, x8, x3
; CHECK-GI-NEXT:    and x0, x0, #0x1
; CHECK-GI-NEXT:    and x2, x2, #0x1
; CHECK-GI-NEXT:    and x3, x22, #0x1
; CHECK-GI-NEXT:    and x12, x12, #0x1
; CHECK-GI-NEXT:    add x0, x0, x2
; CHECK-GI-NEXT:    cset w2, hs
; CHECK-GI-NEXT:    madd x16, x6, x14, x16
; CHECK-GI-NEXT:    adds x8, x8, x17
; CHECK-GI-NEXT:    and x17, x2, #0x1
; CHECK-GI-NEXT:    add x12, x3, x12
; CHECK-GI-NEXT:    add x17, x0, x17
; CHECK-GI-NEXT:    cset w0, hs
; CHECK-GI-NEXT:    add x11, x11, x18
; CHECK-GI-NEXT:    ldr x18, [sp, #24] // 8-byte Reload
; CHECK-GI-NEXT:    ldr x22, [sp, #16] // 8-byte Reload
; CHECK-GI-NEXT:    adds x8, x8, x12
; CHECK-GI-NEXT:    and x12, x28, #0x1
; CHECK-GI-NEXT:    and x0, x0, #0x1
; CHECK-GI-NEXT:    madd x18, x23, x25, x18
; CHECK-GI-NEXT:    add x11, x11, x12
; CHECK-GI-NEXT:    add x12, x17, x0
; CHECK-GI-NEXT:    cset w17, hs
; CHECK-GI-NEXT:    ldr x2, [sp, #40] // 8-byte Reload
; CHECK-GI-NEXT:    madd x3, x22, x5, x2
; CHECK-GI-NEXT:    and x17, x17, #0x1
; CHECK-GI-NEXT:    ldr x21, [sp, #48] // 8-byte Reload
; CHECK-GI-NEXT:    add x11, x11, x15
; CHECK-GI-NEXT:    add x12, x12, x17
; CHECK-GI-NEXT:    madd x6, x24, x1, x27
; CHECK-GI-NEXT:    ldp x17, x15, [sp, #88] // 16-byte Folded Reload
; CHECK-GI-NEXT:    ldr x0, [sp, #104] // 8-byte Reload
; CHECK-GI-NEXT:    add x12, x16, x12
; CHECK-GI-NEXT:    ldr x2, [sp, #72] // 8-byte Reload
; CHECK-GI-NEXT:    madd x13, x19, x14, x13
; CHECK-GI-NEXT:    ldr x16, [sp, #112] // 8-byte Reload
; CHECK-GI-NEXT:    eor x10, x10, x18, asr #63
; CHECK-GI-NEXT:    add x9, x0, x9
; CHECK-GI-NEXT:    add x11, x15, x11
; CHECK-GI-NEXT:    eor x17, x17, x3, asr #63
; CHECK-GI-NEXT:    mul x4, x21, x25
; CHECK-GI-NEXT:    eor x16, x16, x18, asr #63
; CHECK-GI-NEXT:    eor x9, x9, x3, asr #63
; CHECK-GI-NEXT:    eor x0, x26, x6, asr #63
; CHECK-GI-NEXT:    eor x11, x11, x6, asr #63
; CHECK-GI-NEXT:    mul x15, x2, x5
; CHECK-GI-NEXT:    orr x10, x16, x10
; CHECK-GI-NEXT:    orr x9, x17, x9
; CHECK-GI-NEXT:    eor x8, x8, x13, asr #63
; CHECK-GI-NEXT:    eor x12, x12, x13, asr #63
; CHECK-GI-NEXT:    cmp x10, #0
; CHECK-GI-NEXT:    mul x5, x20, x1
; CHECK-GI-NEXT:    orr x11, x0, x11
; CHECK-GI-NEXT:    csel x1, x18, x23, ne
; CHECK-GI-NEXT:    csel x0, x4, x21, ne
; CHECK-GI-NEXT:    cmp x9, #0
; CHECK-GI-NEXT:    orr x8, x8, x12
; CHECK-GI-NEXT:    mul x14, x7, x14
; CHECK-GI-NEXT:    csel x3, x3, x22, ne
; CHECK-GI-NEXT:    csel x2, x15, x2, ne
; CHECK-GI-NEXT:    cmp x11, #0
; CHECK-GI-NEXT:    ldp x22, x21, [sp, #192] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x4, x5, x20, ne
; CHECK-GI-NEXT:    csel x5, x6, x24, ne
; CHECK-GI-NEXT:    cmp x8, #0
; CHECK-GI-NEXT:    ldp x24, x23, [sp, #176] // 16-byte Folded Reload
; CHECK-GI-NEXT:    csel x6, x14, x7, ne
; CHECK-GI-NEXT:    csel x7, x13, x19, ne
; CHECK-GI-NEXT:    ldp x20, x19, [sp, #208] // 16-byte Folded Reload
; CHECK-GI-NEXT:    ldp x26, x25, [sp, #160] // 16-byte Folded Reload
; CHECK-GI-NEXT:    ldp x28, x27, [sp, #144] // 16-byte Folded Reload
; CHECK-GI-NEXT:    ldp x29, x30, [sp, #128] // 16-byte Folded Reload
; CHECK-GI-NEXT:    add sp, sp, #224
; CHECK-GI-NEXT:    ret
  %o = call {<4 x i128>, <4 x i1>} @llvm.smul.with.overflow(<4 x i128> %a, <4 x i128> %b)
  %e0 = extractvalue {<4 x i128>, <4 x i1>} %o, 0
  %e1 = extractvalue {<4 x i128>, <4 x i1>} %o, 1
  %r = select <4 x i1> %e1, <4 x i128> %e0, <4 x i128> %a
  ret <4 x i128> %r
}