; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s

define arm_aapcs_vfpcc void @ssatmul_s_q31(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_s_q31:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT:    .pad #8
; CHECK-NEXT:    sub sp, #8
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq.w .LBB0_8
; CHECK-NEXT:  @ %bb.1: @ %entry
; CHECK-NEXT:    mov r11, r2
; CHECK-NEXT:    cmp r3, #1
; CHECK-NEXT:    bne .LBB0_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    movs r2, #0
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r8, r1
; CHECK-NEXT:    mov r10, r11
; CHECK-NEXT:    b .LBB0_6
; CHECK-NEXT:  .LBB0_3: @ %vector.ph
; CHECK-NEXT:    bic r2, r3, #1
; CHECK-NEXT:    adr r4, .LCPI0_0
; CHECK-NEXT:    subs r7, r2, #2
; CHECK-NEXT:    movs r6, #1
; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
; CHECK-NEXT:    add.w r10, r11, r2, lsl #2
; CHECK-NEXT:    add.w lr, r6, r7, lsr #1
; CHECK-NEXT:    str r2, [sp] @ 4-byte Spill
; CHECK-NEXT:    add.w r8, r1, r2, lsl #2
; CHECK-NEXT:    add.w r12, r0, r2, lsl #2
; CHECK-NEXT:    vldrw.u32 q0, [r4]
; CHECK-NEXT:    vmvn.i32 q1, #0x80000000
; CHECK-NEXT:  .LBB0_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldrd r4, r2, [r0], #8
; CHECK-NEXT:    movs r5, #0
; CHECK-NEXT:    ldrd r7, r6, [r1], #8
; CHECK-NEXT:    smull r4, r7, r7, r4
; CHECK-NEXT:    asrl r4, r7, #31
; CHECK-NEXT:    rsbs.w r9, r4, #-2147483648
; CHECK-NEXT:    mov.w r9, #-1
; CHECK-NEXT:    sbcs.w r3, r9, r7
; CHECK-NEXT:    csetm r3, lt
; CHECK-NEXT:    bfi r5, r3, #0, #8
; CHECK-NEXT:    smull r2, r3, r6, r2
; CHECK-NEXT:    asrl r2, r3, #31
; CHECK-NEXT:    rsbs.w r6, r2, #-2147483648
; CHECK-NEXT:    vmov q2[2], q2[0], r4, r2
; CHECK-NEXT:    sbcs.w r6, r9, r3
; CHECK-NEXT:    vmov q2[3], q2[1], r7, r3
; CHECK-NEXT:    csetm r6, lt
; CHECK-NEXT:    bfi r5, r6, #8, #8
; CHECK-NEXT:    vmsr p0, r5
; CHECK-NEXT:    mvn r5, #-2147483648
; CHECK-NEXT:    vpsel q2, q2, q0
; CHECK-NEXT:    vmov r2, r3, d4
; CHECK-NEXT:    subs r2, r2, r5
; CHECK-NEXT:    sbcs r2, r3, #0
; CHECK-NEXT:    mov.w r3, #0
; CHECK-NEXT:    csetm r2, lt
; CHECK-NEXT:    bfi r3, r2, #0, #8
; CHECK-NEXT:    vmov r2, r4, d5
; CHECK-NEXT:    subs r2, r2, r5
; CHECK-NEXT:    sbcs r2, r4, #0
; CHECK-NEXT:    csetm r2, lt
; CHECK-NEXT:    bfi r3, r2, #8, #8
; CHECK-NEXT:    vmsr p0, r3
; CHECK-NEXT:    vpsel q2, q2, q1
; CHECK-NEXT:    vmov r2, s10
; CHECK-NEXT:    vmov r3, s8
; CHECK-NEXT:    strd r3, r2, [r11], #8
; CHECK-NEXT:    le lr, .LBB0_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    ldrd r2, r3, [sp] @ 8-byte Folded Reload
; CHECK-NEXT:    cmp r2, r3
; CHECK-NEXT:    beq .LBB0_8
; CHECK-NEXT:  .LBB0_6: @ %for.body.preheader
; CHECK-NEXT:    sub.w lr, r3, r2
; CHECK-NEXT:    mov.w r0, #-1
; CHECK-NEXT:    mov.w r1, #-2147483648
; CHECK-NEXT:    mvn r3, #-2147483648
; CHECK-NEXT:  .LBB0_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldr r2, [r12], #4
; CHECK-NEXT:    ldr r4, [r8], #4
; CHECK-NEXT:    smull r2, r5, r4, r2
; CHECK-NEXT:    asrl r2, r5, #31
; CHECK-NEXT:    subs r4, r1, r2
; CHECK-NEXT:    sbcs.w r4, r0, r5
; CHECK-NEXT:    csel r2, r2, r1, lt
; CHECK-NEXT:    csel r4, r5, r0, lt
; CHECK-NEXT:    subs r5, r2, r3
; CHECK-NEXT:    sbcs r4, r4, #0
; CHECK-NEXT:    csel r2, r2, r3, lt
; CHECK-NEXT:    str r2, [r10], #4
; CHECK-NEXT:    le lr, .LBB0_7
; CHECK-NEXT:  .LBB0_8: @ %for.cond.cleanup
; CHECK-NEXT:    add sp, #8
; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
; CHECK-NEXT:    .p2align 4
; CHECK-NEXT:  @ %bb.9:
; CHECK-NEXT:  .LCPI0_0:
; CHECK-NEXT:    .long 2147483648 @ 0x80000000
; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
; CHECK-NEXT:    .long 2147483648 @ 0x80000000
; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
entry:
  switch i32 %N, label %vector.ph [
    i32 0, label %for.cond.cleanup
    i32 1, label %for.body.preheader
  ]

vector.ph:                                        ; preds = %entry
  %n.vec = and i32 %N, -2
  %ind.end = getelementptr i32, ptr %pSrcA, i32 %n.vec
  %ind.end15 = getelementptr i32, ptr %pSrcB, i32 %n.vec
  %ind.end17 = getelementptr i32, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i32, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i32, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i32, ptr %pDst, i32 %index
  %wide.load = load <2 x i32>, ptr %next.gep, align 4
  %0 = sext <2 x i32> %wide.load to <2 x i64>
  %wide.load20 = load <2 x i32>, ptr %next.gep18, align 4
  %1 = sext <2 x i32> %wide.load20 to <2 x i64>
  %2 = mul nsw <2 x i64> %1, %0
  %3 = ashr <2 x i64> %2, <i64 31, i64 31>
  %4 = icmp sgt <2 x i64> %3, <i64 -2147483648, i64 -2147483648>
  %5 = select <2 x i1> %4, <2 x i64> %3, <2 x i64> <i64 -2147483648, i64 -2147483648>
  %6 = icmp slt <2 x i64> %5, <i64 2147483647, i64 2147483647>
  %7 = select <2 x i1> %6, <2 x i64> %5, <2 x i64> <i64 2147483647, i64 2147483647>
  %8 = trunc <2 x i64> %7 to <2 x i32>
  store <2 x i32> %8, ptr %next.gep19, align 4
  %index.next = add i32 %index, 2
  %9 = icmp eq i32 %index.next, %n.vec
  br i1 %9, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry, %middle.block
  %i.012.ph = phi i32 [ 0, %entry ], [ %n.vec, %middle.block ]
  %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %entry ], [ %ind.end, %middle.block ]
  %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %entry ], [ %ind.end15, %middle.block ]
  %pDst.addr.09.ph = phi ptr [ %pDst, %entry ], [ %ind.end17, %middle.block ]
  br label %for.body

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body.preheader, %for.body
  %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader ]
  %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader ]
  %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader ]
  %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader ]
  %incdec.ptr = getelementptr inbounds i32, ptr %pSrcA.addr.011, i32 1
  %10 = load i32, ptr %pSrcA.addr.011, align 4
  %conv = sext i32 %10 to i64
  %incdec.ptr1 = getelementptr inbounds i32, ptr %pSrcB.addr.010, i32 1
  %11 = load i32, ptr %pSrcB.addr.010, align 4
  %conv2 = sext i32 %11 to i64
  %mul = mul nsw i64 %conv2, %conv
  %shr = ashr i64 %mul, 31
  %12 = icmp sgt i64 %shr, -2147483648
  %.val.i = select i1 %12, i64 %shr, i64 -2147483648
  %13 = icmp slt i64 %.val.i, 2147483647
  %retval.0.i = select i1 %13, i64 %.val.i, i64 2147483647
  %conv3 = trunc i64 %retval.0.i to i32
  %incdec.ptr4 = getelementptr inbounds i32, ptr %pDst.addr.09, i32 1
  store i32 %conv3, ptr %pDst.addr.09, align 4
  %inc = add nuw i32 %i.012, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}

define arm_aapcs_vfpcc void @ssatmul_4_q31(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_4_q31:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT:    .pad #4
; CHECK-NEXT:    sub sp, #4
; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
; CHECK-NEXT:    .pad #16
; CHECK-NEXT:    sub sp, #16
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq.w .LBB1_8
; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
; CHECK-NEXT:    mov r5, r1
; CHECK-NEXT:    movs r1, #0
; CHECK-NEXT:    cmp r3, #3
; CHECK-NEXT:    bhi .LBB1_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r9, r5
; CHECK-NEXT:    mov r11, r2
; CHECK-NEXT:    b .LBB1_6
; CHECK-NEXT:  .LBB1_3: @ %vector.ph
; CHECK-NEXT:    bic r1, r3, #3
; CHECK-NEXT:    adr r4, .LCPI1_0
; CHECK-NEXT:    subs r7, r1, #4
; CHECK-NEXT:    movs r6, #1
; CHECK-NEXT:    vldrw.u32 q0, [r4]
; CHECK-NEXT:    adr r4, .LCPI1_1
; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
; CHECK-NEXT:    str r1, [sp] @ 4-byte Spill
; CHECK-NEXT:    add.w r11, r2, r1, lsl #2
; CHECK-NEXT:    add.w r9, r5, r1, lsl #2
; CHECK-NEXT:    add.w r12, r0, r1, lsl #2
; CHECK-NEXT:    vldrw.u32 q1, [r4]
; CHECK-NEXT:  .LBB1_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q3, [r5], #16
; CHECK-NEXT:    vldrw.u32 q2, [r0], #16
; CHECK-NEXT:    str r2, [sp, #12] @ 4-byte Spill
; CHECK-NEXT:    mov.w r2, #-1
; CHECK-NEXT:    vmov.f32 s16, s10
; CHECK-NEXT:    str r5, [sp, #8] @ 4-byte Spill
; CHECK-NEXT:    vmov.f32 s20, s14
; CHECK-NEXT:    mov.w r8, #0
; CHECK-NEXT:    vmov.f32 s18, s11
; CHECK-NEXT:    vmov.f32 s22, s15
; CHECK-NEXT:    vmullb.s32 q6, q5, q4
; CHECK-NEXT:    vmov.f32 s14, s13
; CHECK-NEXT:    vmov r4, r7, d12
; CHECK-NEXT:    asrl r4, r7, #31
; CHECK-NEXT:    vmov.f32 s10, s9
; CHECK-NEXT:    rsbs.w r5, r4, #-2147483648
; CHECK-NEXT:    sbcs.w r5, r2, r7
; CHECK-NEXT:    csetm r5, lt
; CHECK-NEXT:    bfi r8, r5, #0, #8
; CHECK-NEXT:    vmov r10, r5, d13
; CHECK-NEXT:    asrl r10, r5, #31
; CHECK-NEXT:    vmov r6, s14
; CHECK-NEXT:    rsbs.w r3, r10, #-2147483648
; CHECK-NEXT:    vmov q4[2], q4[0], r4, r10
; CHECK-NEXT:    sbcs.w r3, r2, r5
; CHECK-NEXT:    vmov q4[3], q4[1], r7, r5
; CHECK-NEXT:    csetm r3, lt
; CHECK-NEXT:    bfi r8, r3, #8, #8
; CHECK-NEXT:    vmsr p0, r8
; CHECK-NEXT:    mvn r8, #-2147483648
; CHECK-NEXT:    vpsel q4, q4, q0
; CHECK-NEXT:    vmov r3, r4, d8
; CHECK-NEXT:    subs.w r3, r3, r8
; CHECK-NEXT:    sbcs r3, r4, #0
; CHECK-NEXT:    mov.w r4, #0
; CHECK-NEXT:    csetm r3, lt
; CHECK-NEXT:    bfi r4, r3, #0, #8
; CHECK-NEXT:    vmov r3, r5, d9
; CHECK-NEXT:    subs.w r3, r3, r8
; CHECK-NEXT:    sbcs r3, r5, #0
; CHECK-NEXT:    mov.w r5, #0
; CHECK-NEXT:    csetm r3, lt
; CHECK-NEXT:    bfi r4, r3, #8, #8
; CHECK-NEXT:    vmov r3, s8
; CHECK-NEXT:    vmsr p0, r4
; CHECK-NEXT:    vmov r4, s12
; CHECK-NEXT:    vpsel q4, q4, q1
; CHECK-NEXT:    smull r4, r7, r4, r3
; CHECK-NEXT:    asrl r4, r7, #31
; CHECK-NEXT:    rsbs.w r3, r4, #-2147483648
; CHECK-NEXT:    sbcs.w r3, r2, r7
; CHECK-NEXT:    csetm r3, lt
; CHECK-NEXT:    bfi r5, r3, #0, #8
; CHECK-NEXT:    vmov r3, s10
; CHECK-NEXT:    smull r6, r3, r6, r3
; CHECK-NEXT:    asrl r6, r3, #31
; CHECK-NEXT:    rsbs.w r1, r6, #-2147483648
; CHECK-NEXT:    vmov q2[2], q2[0], r4, r6
; CHECK-NEXT:    sbcs.w r1, r2, r3
; CHECK-NEXT:    vmov q2[3], q2[1], r7, r3
; CHECK-NEXT:    csetm r1, lt
; CHECK-NEXT:    bfi r5, r1, #8, #8
; CHECK-NEXT:    vmsr p0, r5
; CHECK-NEXT:    ldrd r5, r2, [sp, #8] @ 8-byte Folded Reload
; CHECK-NEXT:    vpsel q2, q2, q0
; CHECK-NEXT:    vmov r1, r3, d4
; CHECK-NEXT:    subs.w r1, r1, r8
; CHECK-NEXT:    sbcs r1, r3, #0
; CHECK-NEXT:    mov.w r3, #0
; CHECK-NEXT:    csetm r1, lt
; CHECK-NEXT:    bfi r3, r1, #0, #8
; CHECK-NEXT:    vmov r1, r4, d5
; CHECK-NEXT:    subs.w r1, r1, r8
; CHECK-NEXT:    sbcs r1, r4, #0
; CHECK-NEXT:    csetm r1, lt
; CHECK-NEXT:    bfi r3, r1, #8, #8
; CHECK-NEXT:    vmsr p0, r3
; CHECK-NEXT:    vpsel q2, q2, q1
; CHECK-NEXT:    vmov.f32 s9, s10
; CHECK-NEXT:    vmov.f32 s10, s16
; CHECK-NEXT:    vmov.f32 s11, s18
; CHECK-NEXT:    vstrb.8 q2, [r2], #16
; CHECK-NEXT:    le lr, .LBB1_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    ldrd r1, r3, [sp] @ 8-byte Folded Reload
; CHECK-NEXT:    cmp r1, r3
; CHECK-NEXT:    beq .LBB1_8
; CHECK-NEXT:  .LBB1_6: @ %for.body.preheader21
; CHECK-NEXT:    sub.w lr, r3, r1
; CHECK-NEXT:    mov.w r0, #-1
; CHECK-NEXT:    mov.w r3, #-2147483648
; CHECK-NEXT:    mvn r2, #-2147483648
; CHECK-NEXT:  .LBB1_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldr r1, [r12], #4
; CHECK-NEXT:    ldr r4, [r9], #4
; CHECK-NEXT:    smull r4, r1, r4, r1
; CHECK-NEXT:    asrl r4, r1, #31
; CHECK-NEXT:    subs r5, r3, r4
; CHECK-NEXT:    sbcs.w r5, r0, r1
; CHECK-NEXT:    csel r4, r4, r3, lt
; CHECK-NEXT:    csel r1, r1, r0, lt
; CHECK-NEXT:    subs r5, r4, r2
; CHECK-NEXT:    sbcs r1, r1, #0
; CHECK-NEXT:    csel r1, r4, r2, lt
; CHECK-NEXT:    str r1, [r11], #4
; CHECK-NEXT:    le lr, .LBB1_7
; CHECK-NEXT:  .LBB1_8: @ %for.cond.cleanup
; CHECK-NEXT:    add sp, #16
; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
; CHECK-NEXT:    add sp, #4
; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
; CHECK-NEXT:    .p2align 4
; CHECK-NEXT:  @ %bb.9:
; CHECK-NEXT:  .LCPI1_0:
; CHECK-NEXT:    .long 2147483648 @ 0x80000000
; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
; CHECK-NEXT:    .long 2147483648 @ 0x80000000
; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
; CHECK-NEXT:  .LCPI1_1:
; CHECK-NEXT:    .long 2147483647 @ 0x7fffffff
; CHECK-NEXT:    .long 0 @ 0x0
; CHECK-NEXT:    .long 2147483647 @ 0x7fffffff
; CHECK-NEXT:    .long 0 @ 0x0
entry:
  %cmp8 = icmp eq i32 %N, 0
  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry
  %min.iters.check = icmp ult i32 %N, 4
  br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph

for.body.preheader21:                             ; preds = %middle.block, %for.body.preheader
  %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
  %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ]
  %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ]
  %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ]
  br label %for.body

vector.ph:                                        ; preds = %for.body.preheader
  %n.vec = and i32 %N, -4
  %ind.end = getelementptr i32, ptr %pSrcA, i32 %n.vec
  %ind.end15 = getelementptr i32, ptr %pSrcB, i32 %n.vec
  %ind.end17 = getelementptr i32, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i32, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i32, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i32, ptr %pDst, i32 %index
  %wide.load = load <4 x i32>, ptr %next.gep, align 4
  %0 = sext <4 x i32> %wide.load to <4 x i64>
  %wide.load20 = load <4 x i32>, ptr %next.gep18, align 4
  %1 = sext <4 x i32> %wide.load20 to <4 x i64>
  %2 = mul nsw <4 x i64> %1, %0
  %3 = ashr <4 x i64> %2, <i64 31, i64 31, i64 31, i64 31>
  %4 = icmp sgt <4 x i64> %3, <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
  %5 = select <4 x i1> %4, <4 x i64> %3, <4 x i64> <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
  %6 = icmp slt <4 x i64> %5, <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
  %7 = select <4 x i1> %6, <4 x i64> %5, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
  %8 = trunc <4 x i64> %7 to <4 x i32>
  store <4 x i32> %8, ptr %next.gep19, align 4
  %index.next = add i32 %index, 4
  %9 = icmp eq i32 %index.next, %n.vec
  br i1 %9, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body.preheader21, %for.body
  %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ]
  %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ]
  %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ]
  %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ]
  %incdec.ptr = getelementptr inbounds i32, ptr %pSrcA.addr.011, i32 1
  %10 = load i32, ptr %pSrcA.addr.011, align 4
  %conv = sext i32 %10 to i64
  %incdec.ptr1 = getelementptr inbounds i32, ptr %pSrcB.addr.010, i32 1
  %11 = load i32, ptr %pSrcB.addr.010, align 4
  %conv2 = sext i32 %11 to i64
  %mul = mul nsw i64 %conv2, %conv
  %shr = ashr i64 %mul, 31
  %12 = icmp sgt i64 %shr, -2147483648
  %.val.i = select i1 %12, i64 %shr, i64 -2147483648
  %13 = icmp slt i64 %.val.i, 2147483647
  %retval.0.i = select i1 %13, i64 %.val.i, i64 2147483647
  %conv3 = trunc i64 %retval.0.i to i32
  %incdec.ptr4 = getelementptr inbounds i32, ptr %pDst.addr.09, i32 1
  store i32 %conv3, ptr %pDst.addr.09, align 4
  %inc = add nuw i32 %i.012, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}

define arm_aapcs_vfpcc void @ssatmul_4t_q31(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_4t_q31:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    .pad #24
; CHECK-NEXT:    sub sp, #24
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq.w .LBB2_3
; CHECK-NEXT:  @ %bb.1: @ %vector.ph
; CHECK-NEXT:    adds r6, r3, #3
; CHECK-NEXT:    movs r5, #1
; CHECK-NEXT:    bic r6, r6, #3
; CHECK-NEXT:    adr r4, .LCPI2_1
; CHECK-NEXT:    subs r6, #4
; CHECK-NEXT:    vldrw.u32 q2, [r4]
; CHECK-NEXT:    mov.w r9, #0
; CHECK-NEXT:    mov.w r12, #-1
; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
; CHECK-NEXT:    adr r5, .LCPI2_0
; CHECK-NEXT:    vldrw.u32 q0, [r5]
; CHECK-NEXT:    adr r5, .LCPI2_2
; CHECK-NEXT:    subs r6, r3, #1
; CHECK-NEXT:    vldrw.u32 q3, [r5]
; CHECK-NEXT:    vdup.32 q1, r6
; CHECK-NEXT:    mvn r8, #-2147483648
; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
; CHECK-NEXT:  .LBB2_2: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
; CHECK-NEXT:    vdup.32 q4, r9
; CHECK-NEXT:    movs r4, #0
; CHECK-NEXT:    add.w r9, r9, #4
; CHECK-NEXT:    vorr q4, q4, q0
; CHECK-NEXT:    vcmp.u32 cs, q1, q4
; CHECK-NEXT:    vstr p0, [sp, #20] @ 4-byte Spill
; CHECK-NEXT:    vpstt
; CHECK-NEXT:    vldrwt.u32 q4, [r0], #16
; CHECK-NEXT:    vldrwt.u32 q5, [r1], #16
; CHECK-NEXT:    vmov.f32 s24, s18
; CHECK-NEXT:    vmov.f32 s26, s19
; CHECK-NEXT:    vmov.f32 s28, s22
; CHECK-NEXT:    vmov.f32 s30, s23
; CHECK-NEXT:    vmullb.s32 q0, q7, q6
; CHECK-NEXT:    vmov.f32 s18, s21
; CHECK-NEXT:    vmov r10, r5, d0
; CHECK-NEXT:    asrl r10, r5, #31
; CHECK-NEXT:    rsbs.w r7, r10, #-2147483648
; CHECK-NEXT:    sbcs.w r7, r12, r5
; CHECK-NEXT:    csetm r7, lt
; CHECK-NEXT:    bfi r4, r7, #0, #8
; CHECK-NEXT:    vmov r6, r7, d1
; CHECK-NEXT:    asrl r6, r7, #31
; CHECK-NEXT:    rsbs.w r3, r6, #-2147483648
; CHECK-NEXT:    vmov q0[2], q0[0], r10, r6
; CHECK-NEXT:    sbcs.w r3, r12, r7
; CHECK-NEXT:    vmov q0[3], q0[1], r5, r7
; CHECK-NEXT:    csetm r3, lt
; CHECK-NEXT:    vmov r7, s18
; CHECK-NEXT:    bfi r4, r3, #8, #8
; CHECK-NEXT:    vmsr p0, r4
; CHECK-NEXT:    vpsel q0, q0, q2
; CHECK-NEXT:    vmov r3, r4, d0
; CHECK-NEXT:    subs.w r3, r3, r8
; CHECK-NEXT:    sbcs r3, r4, #0
; CHECK-NEXT:    mov.w r4, #0
; CHECK-NEXT:    csetm r3, lt
; CHECK-NEXT:    bfi r4, r3, #0, #8
; CHECK-NEXT:    vmov r3, r5, d1
; CHECK-NEXT:    subs.w r3, r3, r8
; CHECK-NEXT:    sbcs r3, r5, #0
; CHECK-NEXT:    csetm r3, lt
; CHECK-NEXT:    bfi r4, r3, #8, #8
; CHECK-NEXT:    vmov r3, s16
; CHECK-NEXT:    vmsr p0, r4
; CHECK-NEXT:    vmov r4, s20
; CHECK-NEXT:    vpsel q6, q0, q3
; CHECK-NEXT:    vmov.f32 s2, s17
; CHECK-NEXT:    smull r10, r5, r4, r3
; CHECK-NEXT:    movs r4, #0
; CHECK-NEXT:    asrl r10, r5, #31
; CHECK-NEXT:    rsbs.w r3, r10, #-2147483648
; CHECK-NEXT:    sbcs.w r3, r12, r5
; CHECK-NEXT:    csetm r3, lt
; CHECK-NEXT:    bfi r4, r3, #0, #8
; CHECK-NEXT:    vmov r3, s2
; CHECK-NEXT:    smull r6, r3, r7, r3
; CHECK-NEXT:    asrl r6, r3, #31
; CHECK-NEXT:    rsbs.w r7, r6, #-2147483648
; CHECK-NEXT:    vmov q0[2], q0[0], r10, r6
; CHECK-NEXT:    sbcs.w r7, r12, r3
; CHECK-NEXT:    vmov q0[3], q0[1], r5, r3
; CHECK-NEXT:    csetm r7, lt
; CHECK-NEXT:    bfi r4, r7, #8, #8
; CHECK-NEXT:    vmsr p0, r4
; CHECK-NEXT:    vpsel q0, q0, q2
; CHECK-NEXT:    vmov r3, r4, d0
; CHECK-NEXT:    subs.w r3, r3, r8
; CHECK-NEXT:    sbcs r3, r4, #0
; CHECK-NEXT:    mov.w r4, #0
; CHECK-NEXT:    csetm r3, lt
; CHECK-NEXT:    bfi r4, r3, #0, #8
; CHECK-NEXT:    vmov r3, r5, d1
; CHECK-NEXT:    subs.w r3, r3, r8
; CHECK-NEXT:    sbcs r3, r5, #0
; CHECK-NEXT:    csetm r3, lt
; CHECK-NEXT:    bfi r4, r3, #8, #8
; CHECK-NEXT:    vmsr p0, r4
; CHECK-NEXT:    vpsel q0, q0, q3
; CHECK-NEXT:    vldr p0, [sp, #20] @ 4-byte Reload
; CHECK-NEXT:    vmov.f32 s1, s2
; CHECK-NEXT:    vmov.f32 s2, s24
; CHECK-NEXT:    vmov.f32 s3, s26
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrwt.32 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB2_2
; CHECK-NEXT:  .LBB2_3: @ %for.cond.cleanup
; CHECK-NEXT:    add sp, #24
; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
; CHECK-NEXT:    .p2align 4
; CHECK-NEXT:  @ %bb.4:
; CHECK-NEXT:  .LCPI2_0:
; CHECK-NEXT:    .long 0 @ 0x0
; CHECK-NEXT:    .long 1 @ 0x1
; CHECK-NEXT:    .long 2 @ 0x2
; CHECK-NEXT:    .long 3 @ 0x3
; CHECK-NEXT:  .LCPI2_1:
; CHECK-NEXT:    .long 2147483648 @ 0x80000000
; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
; CHECK-NEXT:    .long 2147483648 @ 0x80000000
; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
; CHECK-NEXT:  .LCPI2_2:
; CHECK-NEXT:    .long 2147483647 @ 0x7fffffff
; CHECK-NEXT:    .long 0 @ 0x0
; CHECK-NEXT:    .long 2147483647 @ 0x7fffffff
; CHECK-NEXT:    .long 0 @ 0x0
entry:
  %cmp8 = icmp eq i32 %N, 0
  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph

vector.ph:                                        ; preds = %entry
  %n.rnd.up = add i32 %N, 3
  %n.vec = and i32 %n.rnd.up, -4
  %trip.count.minus.1 = add i32 %N, -1
  %broadcast.splatinsert20 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
  %broadcast.splat21 = shufflevector <4 x i32> %broadcast.splatinsert20, <4 x i32> undef, <4 x i32> zeroinitializer
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
  %induction = or <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
  %next.gep = getelementptr i32, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i32, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i32, ptr %pDst, i32 %index
  %0 = icmp ule <4 x i32> %induction, %broadcast.splat21
  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %next.gep, i32 4, <4 x i1> %0, <4 x i32> undef)
  %1 = sext <4 x i32> %wide.masked.load to <4 x i64>
  %wide.masked.load22 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %next.gep18, i32 4, <4 x i1> %0, <4 x i32> undef)
  %2 = sext <4 x i32> %wide.masked.load22 to <4 x i64>
  %3 = mul nsw <4 x i64> %2, %1
  %4 = ashr <4 x i64> %3, <i64 31, i64 31, i64 31, i64 31>
  %5 = icmp sgt <4 x i64> %4, <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
  %6 = select <4 x i1> %5, <4 x i64> %4, <4 x i64> <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
  %7 = icmp slt <4 x i64> %6, <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
  %8 = select <4 x i1> %7, <4 x i64> %6, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
  %9 = trunc <4 x i64> %8 to <4 x i32>
  call void @llvm.masked.store.v4i32.p0(<4 x i32> %9, ptr %next.gep19, i32 4, <4 x i1> %0)
  %index.next = add i32 %index, 4
  %10 = icmp eq i32 %index.next, %n.vec
  br i1 %10, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body, %entry
  ret void
}

define arm_aapcs_vfpcc void @usatmul_2_q31(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: usatmul_2_q31:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT:    .pad #4
; CHECK-NEXT:    sub sp, #4
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq .LBB3_8
; CHECK-NEXT:  @ %bb.1: @ %entry
; CHECK-NEXT:    mov r8, r2
; CHECK-NEXT:    cmp r3, #1
; CHECK-NEXT:    bne .LBB3_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    movs r7, #0
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r11, r1
; CHECK-NEXT:    mov r2, r8
; CHECK-NEXT:    b .LBB3_6
; CHECK-NEXT:  .LBB3_3: @ %vector.ph
; CHECK-NEXT:    bic r5, r3, #1
; CHECK-NEXT:    movs r6, #1
; CHECK-NEXT:    subs r7, r5, #2
; CHECK-NEXT:    str r5, [sp] @ 4-byte Spill
; CHECK-NEXT:    add.w r2, r8, r5, lsl #2
; CHECK-NEXT:    add.w r11, r1, r5, lsl #2
; CHECK-NEXT:    add.w lr, r6, r7, lsr #1
; CHECK-NEXT:    add.w r12, r0, r5, lsl #2
; CHECK-NEXT:    vmov.i8 q0, #0xff
; CHECK-NEXT:  .LBB3_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldrd r4, r9, [r0], #8
; CHECK-NEXT:    ldrd r5, r10, [r1], #8
; CHECK-NEXT:    umull r4, r5, r5, r4
; CHECK-NEXT:    lsrl r4, r5, #31
; CHECK-NEXT:    subs.w r6, r4, #-1
; CHECK-NEXT:    sbcs r5, r5, #0
; CHECK-NEXT:    mov.w r6, #0
; CHECK-NEXT:    csetm r5, lo
; CHECK-NEXT:    bfi r6, r5, #0, #8
; CHECK-NEXT:    umull r10, r5, r10, r9
; CHECK-NEXT:    lsrl r10, r5, #31
; CHECK-NEXT:    subs.w r7, r10, #-1
; CHECK-NEXT:    vmov q1[2], q1[0], r4, r10
; CHECK-NEXT:    sbcs r5, r5, #0
; CHECK-NEXT:    csetm r5, lo
; CHECK-NEXT:    bfi r6, r5, #8, #8
; CHECK-NEXT:    vmsr p0, r6
; CHECK-NEXT:    vpsel q1, q1, q0
; CHECK-NEXT:    vmov r4, s6
; CHECK-NEXT:    vmov r5, s4
; CHECK-NEXT:    strd r5, r4, [r8], #8
; CHECK-NEXT:    le lr, .LBB3_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    ldr r7, [sp] @ 4-byte Reload
; CHECK-NEXT:    cmp r7, r3
; CHECK-NEXT:    beq .LBB3_8
; CHECK-NEXT:  .LBB3_6: @ %for.body.preheader
; CHECK-NEXT:    sub.w lr, r3, r7
; CHECK-NEXT:  .LBB3_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldr r0, [r12], #4
; CHECK-NEXT:    ldr r1, [r11], #4
; CHECK-NEXT:    umull r0, r1, r1, r0
; CHECK-NEXT:    lsrl r0, r1, #31
; CHECK-NEXT:    subs.w r3, r0, #-1
; CHECK-NEXT:    sbcs r1, r1, #0
; CHECK-NEXT:    it hs
; CHECK-NEXT:    movhs.w r0, #-1
; CHECK-NEXT:    str r0, [r2], #4
; CHECK-NEXT:    le lr, .LBB3_7
; CHECK-NEXT:  .LBB3_8: @ %for.cond.cleanup
; CHECK-NEXT:    add sp, #4
; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
entry:
  switch i32 %N, label %vector.ph [
    i32 0, label %for.cond.cleanup
    i32 1, label %for.body.preheader
  ]

vector.ph:                                        ; preds = %entry
  %n.vec = and i32 %N, -2
  %ind.end = getelementptr i32, ptr %pSrcA, i32 %n.vec
  %ind.end15 = getelementptr i32, ptr %pSrcB, i32 %n.vec
  %ind.end17 = getelementptr i32, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i32, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i32, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i32, ptr %pDst, i32 %index
  %wide.load = load <2 x i32>, ptr %next.gep, align 4
  %0 = zext <2 x i32> %wide.load to <2 x i64>
  %wide.load20 = load <2 x i32>, ptr %next.gep18, align 4
  %1 = zext <2 x i32> %wide.load20 to <2 x i64>
  %2 = mul nuw <2 x i64> %1, %0
  %3 = lshr <2 x i64> %2, <i64 31, i64 31>
  %4 = icmp ult <2 x i64> %3, <i64 4294967295, i64 4294967295>
  %5 = select <2 x i1> %4, <2 x i64> %3, <2 x i64> <i64 4294967295, i64 4294967295>
  %6 = trunc <2 x i64> %5 to <2 x i32>
  store <2 x i32> %6, ptr %next.gep19, align 4
  %index.next = add i32 %index, 2
  %7 = icmp eq i32 %index.next, %n.vec
  br i1 %7, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry, %middle.block
  %i.012.ph = phi i32 [ 0, %entry ], [ %n.vec, %middle.block ]
  %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %entry ], [ %ind.end, %middle.block ]
  %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %entry ], [ %ind.end15, %middle.block ]
  %pDst.addr.09.ph = phi ptr [ %pDst, %entry ], [ %ind.end17, %middle.block ]
  br label %for.body

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body.preheader, %for.body
  %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader ]
  %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader ]
  %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader ]
  %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader ]
  %incdec.ptr = getelementptr inbounds i32, ptr %pSrcA.addr.011, i32 1
  %8 = load i32, ptr %pSrcA.addr.011, align 4
  %conv = zext i32 %8 to i64
  %incdec.ptr1 = getelementptr inbounds i32, ptr %pSrcB.addr.010, i32 1
  %9 = load i32, ptr %pSrcB.addr.010, align 4
  %conv2 = zext i32 %9 to i64
  %mul = mul nuw i64 %conv2, %conv
  %shr = lshr i64 %mul, 31
  %10 = icmp ult i64 %shr, 4294967295
  %retval.0.i = select i1 %10, i64 %shr, i64 4294967295
  %conv3 = trunc i64 %retval.0.i to i32
  %incdec.ptr4 = getelementptr inbounds i32, ptr %pDst.addr.09, i32 1
  store i32 %conv3, ptr %pDst.addr.09, align 4
  %inc = add nuw i32 %i.012, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}

define arm_aapcs_vfpcc void @usatmul_4_q31(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: usatmul_4_q31:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT:    .pad #4
; CHECK-NEXT:    sub sp, #4
; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
; CHECK-NEXT:    vpush {d8, d9, d10, d11}
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq.w .LBB4_8
; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
; CHECK-NEXT:    mov.w r8, #0
; CHECK-NEXT:    cmp r3, #3
; CHECK-NEXT:    bhi .LBB4_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r9, r1
; CHECK-NEXT:    mov r11, r2
; CHECK-NEXT:    b .LBB4_6
; CHECK-NEXT:  .LBB4_3: @ %vector.ph
; CHECK-NEXT:    bic r8, r3, #3
; CHECK-NEXT:    movs r6, #1
; CHECK-NEXT:    sub.w r7, r8, #4
; CHECK-NEXT:    vmov.i64 q0, #0xffffffff
; CHECK-NEXT:    add.w r11, r2, r8, lsl #2
; CHECK-NEXT:    add.w r9, r1, r8, lsl #2
; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
; CHECK-NEXT:    add.w r12, r0, r8, lsl #2
; CHECK-NEXT:  .LBB4_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
; CHECK-NEXT:    vldrw.u32 q2, [r1], #16
; CHECK-NEXT:    vmov.f32 s12, s6
; CHECK-NEXT:    vmov.f32 s14, s7
; CHECK-NEXT:    vmov.f32 s16, s10
; CHECK-NEXT:    vmov.f32 s18, s11
; CHECK-NEXT:    vmullb.u32 q5, q4, q3
; CHECK-NEXT:    vmov.f32 s6, s5
; CHECK-NEXT:    vmov r10, r5, d10
; CHECK-NEXT:    lsrl r10, r5, #31
; CHECK-NEXT:    vmov.f32 s10, s9
; CHECK-NEXT:    subs.w r6, r10, #-1
; CHECK-NEXT:    sbcs r5, r5, #0
; CHECK-NEXT:    mov.w r6, #0
; CHECK-NEXT:    csetm r5, lo
; CHECK-NEXT:    vmullb.u32 q4, q2, q1
; CHECK-NEXT:    bfi r6, r5, #0, #8
; CHECK-NEXT:    vmov r4, r5, d11
; CHECK-NEXT:    lsrl r4, r5, #31
; CHECK-NEXT:    subs.w r7, r4, #-1
; CHECK-NEXT:    vmov q3[2], q3[0], r10, r4
; CHECK-NEXT:    sbcs r5, r5, #0
; CHECK-NEXT:    csetm r5, lo
; CHECK-NEXT:    bfi r6, r5, #8, #8
; CHECK-NEXT:    vmov r10, r5, d8
; CHECK-NEXT:    lsrl r10, r5, #31
; CHECK-NEXT:    vmsr p0, r6
; CHECK-NEXT:    subs.w r6, r10, #-1
; CHECK-NEXT:    vpsel q3, q3, q0
; CHECK-NEXT:    sbcs r5, r5, #0
; CHECK-NEXT:    mov.w r6, #0
; CHECK-NEXT:    csetm r5, lo
; CHECK-NEXT:    bfi r6, r5, #0, #8
; CHECK-NEXT:    vmov r4, r5, d9
; CHECK-NEXT:    lsrl r4, r5, #31
; CHECK-NEXT:    subs.w r7, r4, #-1
; CHECK-NEXT:    vmov q1[2], q1[0], r10, r4
; CHECK-NEXT:    sbcs r5, r5, #0
; CHECK-NEXT:    csetm r5, lo
; CHECK-NEXT:    bfi r6, r5, #8, #8
; CHECK-NEXT:    vmsr p0, r6
; CHECK-NEXT:    vpsel q1, q1, q0
; CHECK-NEXT:    vmov.f32 s5, s6
; CHECK-NEXT:    vmov.f32 s6, s12
; CHECK-NEXT:    vmov.f32 s7, s14
; CHECK-NEXT:    vstrb.8 q1, [r2], #16
; CHECK-NEXT:    le lr, .LBB4_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    cmp r8, r3
; CHECK-NEXT:    beq .LBB4_8
; CHECK-NEXT:  .LBB4_6: @ %for.body.preheader21
; CHECK-NEXT:    sub.w lr, r3, r8
; CHECK-NEXT:  .LBB4_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldr r0, [r12], #4
; CHECK-NEXT:    ldr r1, [r9], #4
; CHECK-NEXT:    umull r0, r1, r1, r0
; CHECK-NEXT:    lsrl r0, r1, #31
; CHECK-NEXT:    subs.w r2, r0, #-1
; CHECK-NEXT:    sbcs r1, r1, #0
; CHECK-NEXT:    it hs
; CHECK-NEXT:    movhs.w r0, #-1
; CHECK-NEXT:    str r0, [r11], #4
; CHECK-NEXT:    le lr, .LBB4_7
; CHECK-NEXT:  .LBB4_8: @ %for.cond.cleanup
; CHECK-NEXT:    vpop {d8, d9, d10, d11}
; CHECK-NEXT:    add sp, #4
; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
entry:
  %cmp8 = icmp eq i32 %N, 0
  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry
  %min.iters.check = icmp ult i32 %N, 4
  br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph

for.body.preheader21:                             ; preds = %middle.block, %for.body.preheader
  %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
  %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ]
  %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ]
  %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ]
  br label %for.body

vector.ph:                                        ; preds = %for.body.preheader
  %n.vec = and i32 %N, -4
  %ind.end = getelementptr i32, ptr %pSrcA, i32 %n.vec
  %ind.end15 = getelementptr i32, ptr %pSrcB, i32 %n.vec
  %ind.end17 = getelementptr i32, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i32, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i32, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i32, ptr %pDst, i32 %index
  %wide.load = load <4 x i32>, ptr %next.gep, align 4
  %0 = zext <4 x i32> %wide.load to <4 x i64>
  %wide.load20 = load <4 x i32>, ptr %next.gep18, align 4
  %1 = zext <4 x i32> %wide.load20 to <4 x i64>
  %2 = mul nuw <4 x i64> %1, %0
  %3 = lshr <4 x i64> %2, <i64 31, i64 31, i64 31, i64 31>
  %4 = icmp ult <4 x i64> %3, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
  %5 = select <4 x i1> %4, <4 x i64> %3, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
  %6 = trunc <4 x i64> %5 to <4 x i32>
  store <4 x i32> %6, ptr %next.gep19, align 4
  %index.next = add i32 %index, 4
  %7 = icmp eq i32 %index.next, %n.vec
  br i1 %7, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body.preheader21, %for.body
  %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ]
  %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ]
  %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ]
  %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ]
  %incdec.ptr = getelementptr inbounds i32, ptr %pSrcA.addr.011, i32 1
  %8 = load i32, ptr %pSrcA.addr.011, align 4
  %conv = zext i32 %8 to i64
  %incdec.ptr1 = getelementptr inbounds i32, ptr %pSrcB.addr.010, i32 1
  %9 = load i32, ptr %pSrcB.addr.010, align 4
  %conv2 = zext i32 %9 to i64
  %mul = mul nuw i64 %conv2, %conv
  %shr = lshr i64 %mul, 31
  %10 = icmp ult i64 %shr, 4294967295
  %retval.0.i = select i1 %10, i64 %shr, i64 4294967295
  %conv3 = trunc i64 %retval.0.i to i32
  %incdec.ptr4 = getelementptr inbounds i32, ptr %pDst.addr.09, i32 1
  store i32 %conv3, ptr %pDst.addr.09, align 4
  %inc = add nuw i32 %i.012, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}


; i16

define arm_aapcs_vfpcc void @ssatmul_4_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_4_q15:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, lr}
; CHECK-NEXT:    push {r4, r5, r6, lr}
; CHECK-NEXT:    cbz r3, .LBB5_8
; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
; CHECK-NEXT:    cmp r3, #3
; CHECK-NEXT:    bhi .LBB5_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    movs r5, #0
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r6, r1
; CHECK-NEXT:    mov r4, r2
; CHECK-NEXT:    b .LBB5_6
; CHECK-NEXT:  .LBB5_3: @ %vector.ph
; CHECK-NEXT:    bic r5, r3, #3
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    subs r6, r5, #4
; CHECK-NEXT:    add.w r12, r0, r5, lsl #1
; CHECK-NEXT:    add.w lr, r4, r6, lsr #2
; CHECK-NEXT:    add.w r4, r2, r5, lsl #1
; CHECK-NEXT:    add.w r6, r1, r5, lsl #1
; CHECK-NEXT:  .LBB5_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.s32 q0, [r0], #8
; CHECK-NEXT:    vldrh.s32 q1, [r1], #8
; CHECK-NEXT:    vmul.i32 q0, q1, q0
; CHECK-NEXT:    vqshrnb.s32 q0, q0, #15
; CHECK-NEXT:    vstrh.32 q0, [r2], #8
; CHECK-NEXT:    le lr, .LBB5_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    cmp r5, r3
; CHECK-NEXT:    it eq
; CHECK-NEXT:    popeq {r4, r5, r6, pc}
; CHECK-NEXT:  .LBB5_6: @ %for.body.preheader21
; CHECK-NEXT:    sub.w lr, r3, r5
; CHECK-NEXT:  .LBB5_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldrsh r0, [r12], #2
; CHECK-NEXT:    ldrsh r1, [r6], #2
; CHECK-NEXT:    muls r0, r1, r0
; CHECK-NEXT:    ssat r0, #16, r0, asr #15
; CHECK-NEXT:    strh r0, [r4], #2
; CHECK-NEXT:    le lr, .LBB5_7
; CHECK-NEXT:  .LBB5_8: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r4, r5, r6, pc}
entry:
  %cmp8 = icmp eq i32 %N, 0
  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry
  %min.iters.check = icmp ult i32 %N, 4
  br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph

for.body.preheader21:                             ; preds = %middle.block, %for.body.preheader
  %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
  %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ]
  %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ]
  %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ]
  br label %for.body

vector.ph:                                        ; preds = %for.body.preheader
  %n.vec = and i32 %N, -4
  %ind.end = getelementptr i16, ptr %pSrcA, i32 %n.vec
  %ind.end15 = getelementptr i16, ptr %pSrcB, i32 %n.vec
  %ind.end17 = getelementptr i16, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i16, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i16, ptr %pDst, i32 %index
  %wide.load = load <4 x i16>, ptr %next.gep, align 2
  %0 = sext <4 x i16> %wide.load to <4 x i32>
  %wide.load20 = load <4 x i16>, ptr %next.gep18, align 2
  %1 = sext <4 x i16> %wide.load20 to <4 x i32>
  %2 = mul nsw <4 x i32> %1, %0
  %3 = ashr <4 x i32> %2, <i32 15, i32 15, i32 15, i32 15>
  %4 = icmp sgt <4 x i32> %3, <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %5 = select <4 x i1> %4, <4 x i32> %3, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %6 = icmp slt <4 x i32> %5, <i32 32767, i32 32767, i32 32767, i32 32767>
  %7 = select <4 x i1> %6, <4 x i32> %5, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
  %8 = trunc <4 x i32> %7 to <4 x i16>
  store <4 x i16> %8, ptr %next.gep19, align 2
  %index.next = add i32 %index, 4
  %9 = icmp eq i32 %index.next, %n.vec
  br i1 %9, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body.preheader21, %for.body
  %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ]
  %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ]
  %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ]
  %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ]
  %incdec.ptr = getelementptr inbounds i16, ptr %pSrcA.addr.011, i32 1
  %10 = load i16, ptr %pSrcA.addr.011, align 2
  %conv = sext i16 %10 to i32
  %incdec.ptr1 = getelementptr inbounds i16, ptr %pSrcB.addr.010, i32 1
  %11 = load i16, ptr %pSrcB.addr.010, align 2
  %conv2 = sext i16 %11 to i32
  %mul = mul nsw i32 %conv2, %conv
  %shr = ashr i32 %mul, 15
  %12 = icmp sgt i32 %shr, -32768
  %.val.i = select i1 %12, i32 %shr, i32 -32768
  %13 = icmp slt i32 %.val.i, 32767
  %retval.0.i = select i1 %13, i32 %.val.i, i32 32767
  %conv3 = trunc i32 %retval.0.i to i16
  %incdec.ptr4 = getelementptr inbounds i16, ptr %pDst.addr.09, i32 1
  store i16 %conv3, ptr %pDst.addr.09, align 2
  %inc = add nuw i32 %i.012, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}

define arm_aapcs_vfpcc void @ssatmul_8_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_8_q15:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, lr}
; CHECK-NEXT:    push {r4, r5, r6, lr}
; CHECK-NEXT:    cbz r3, .LBB6_8
; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
; CHECK-NEXT:    cmp r3, #7
; CHECK-NEXT:    bhi .LBB6_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    movs r5, #0
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r6, r1
; CHECK-NEXT:    mov r4, r2
; CHECK-NEXT:    b .LBB6_6
; CHECK-NEXT:  .LBB6_3: @ %vector.ph
; CHECK-NEXT:    bic r5, r3, #7
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    sub.w r6, r5, #8
; CHECK-NEXT:    add.w r12, r0, r5, lsl #1
; CHECK-NEXT:    add.w lr, r4, r6, lsr #3
; CHECK-NEXT:    add.w r4, r2, r5, lsl #1
; CHECK-NEXT:    add.w r6, r1, r5, lsl #1
; CHECK-NEXT:  .LBB6_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
; CHECK-NEXT:    vmullt.s16 q2, q1, q0
; CHECK-NEXT:    vmullb.s16 q0, q1, q0
; CHECK-NEXT:    vqshrnb.s32 q0, q0, #15
; CHECK-NEXT:    vqshrnt.s32 q0, q2, #15
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB6_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    cmp r5, r3
; CHECK-NEXT:    it eq
; CHECK-NEXT:    popeq {r4, r5, r6, pc}
; CHECK-NEXT:  .LBB6_6: @ %for.body.preheader21
; CHECK-NEXT:    sub.w lr, r3, r5
; CHECK-NEXT:  .LBB6_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldrsh r0, [r12], #2
; CHECK-NEXT:    ldrsh r1, [r6], #2
; CHECK-NEXT:    muls r0, r1, r0
; CHECK-NEXT:    ssat r0, #16, r0, asr #15
; CHECK-NEXT:    strh r0, [r4], #2
; CHECK-NEXT:    le lr, .LBB6_7
; CHECK-NEXT:  .LBB6_8: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r4, r5, r6, pc}
entry:
  %cmp8 = icmp eq i32 %N, 0
  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry
  %min.iters.check = icmp ult i32 %N, 8
  br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph

for.body.preheader21:                             ; preds = %middle.block, %for.body.preheader
  %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
  %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ]
  %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ]
  %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ]
  br label %for.body

vector.ph:                                        ; preds = %for.body.preheader
  %n.vec = and i32 %N, -8
  %ind.end = getelementptr i16, ptr %pSrcA, i32 %n.vec
  %ind.end15 = getelementptr i16, ptr %pSrcB, i32 %n.vec
  %ind.end17 = getelementptr i16, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i16, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i16, ptr %pDst, i32 %index
  %wide.load = load <8 x i16>, ptr %next.gep, align 2
  %0 = sext <8 x i16> %wide.load to <8 x i32>
  %wide.load20 = load <8 x i16>, ptr %next.gep18, align 2
  %1 = sext <8 x i16> %wide.load20 to <8 x i32>
  %2 = mul nsw <8 x i32> %1, %0
  %3 = ashr <8 x i32> %2, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
  %4 = icmp sgt <8 x i32> %3, <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %5 = select <8 x i1> %4, <8 x i32> %3, <8 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %6 = icmp slt <8 x i32> %5, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
  %7 = select <8 x i1> %6, <8 x i32> %5, <8 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
  %8 = trunc <8 x i32> %7 to <8 x i16>
  store <8 x i16> %8, ptr %next.gep19, align 2
  %index.next = add i32 %index, 8
  %9 = icmp eq i32 %index.next, %n.vec
  br i1 %9, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body.preheader21, %for.body
  %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ]
  %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ]
  %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ]
  %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ]
  %incdec.ptr = getelementptr inbounds i16, ptr %pSrcA.addr.011, i32 1
  %10 = load i16, ptr %pSrcA.addr.011, align 2
  %conv = sext i16 %10 to i32
  %incdec.ptr1 = getelementptr inbounds i16, ptr %pSrcB.addr.010, i32 1
  %11 = load i16, ptr %pSrcB.addr.010, align 2
  %conv2 = sext i16 %11 to i32
  %mul = mul nsw i32 %conv2, %conv
  %shr = ashr i32 %mul, 15
  %12 = icmp sgt i32 %shr, -32768
  %.val.i = select i1 %12, i32 %shr, i32 -32768
  %13 = icmp slt i32 %.val.i, 32767
  %retval.0.i = select i1 %13, i32 %.val.i, i32 32767
  %conv3 = trunc i32 %retval.0.i to i16
  %incdec.ptr4 = getelementptr inbounds i16, ptr %pDst.addr.09, i32 1
  store i16 %conv3, ptr %pDst.addr.09, align 2
  %inc = add nuw i32 %i.012, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}

define arm_aapcs_vfpcc void @ssatmul_8i_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_8i_q15:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, lr}
; CHECK-NEXT:    push {r4, r5, r6, lr}
; CHECK-NEXT:    cbz r3, .LBB7_8
; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
; CHECK-NEXT:    cmp r3, #7
; CHECK-NEXT:    bhi .LBB7_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    movs r5, #0
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r6, r1
; CHECK-NEXT:    mov r4, r2
; CHECK-NEXT:    b .LBB7_6
; CHECK-NEXT:  .LBB7_3: @ %vector.ph
; CHECK-NEXT:    bic r5, r3, #7
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    sub.w r6, r5, #8
; CHECK-NEXT:    add.w r12, r0, r5, lsl #1
; CHECK-NEXT:    add.w lr, r4, r6, lsr #3
; CHECK-NEXT:    add.w r4, r2, r5, lsl #1
; CHECK-NEXT:    add.w r6, r1, r5, lsl #1
; CHECK-NEXT:  .LBB7_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
; CHECK-NEXT:    vmullt.s16 q2, q1, q0
; CHECK-NEXT:    vmullb.s16 q0, q1, q0
; CHECK-NEXT:    vqshrnb.s32 q0, q0, #15
; CHECK-NEXT:    vqshrnt.s32 q0, q2, #15
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB7_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    cmp r5, r3
; CHECK-NEXT:    it eq
; CHECK-NEXT:    popeq {r4, r5, r6, pc}
; CHECK-NEXT:  .LBB7_6: @ %for.body.preheader21
; CHECK-NEXT:    sub.w lr, r3, r5
; CHECK-NEXT:  .LBB7_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldrsh r0, [r12], #2
; CHECK-NEXT:    ldrsh r1, [r6], #2
; CHECK-NEXT:    muls r0, r1, r0
; CHECK-NEXT:    ssat r0, #16, r0, asr #15
; CHECK-NEXT:    strh r0, [r4], #2
; CHECK-NEXT:    le lr, .LBB7_7
; CHECK-NEXT:  .LBB7_8: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r4, r5, r6, pc}
entry:
  %cmp8 = icmp eq i32 %N, 0
  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry
  %min.iters.check = icmp ult i32 %N, 8
  br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph

for.body.preheader21:                             ; preds = %middle.block, %for.body.preheader
  %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
  %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ]
  %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ]
  %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ]
  br label %for.body

vector.ph:                                        ; preds = %for.body.preheader
  %n.vec = and i32 %N, -8
  %ind.end = getelementptr i16, ptr %pSrcA, i32 %n.vec
  %ind.end15 = getelementptr i16, ptr %pSrcB, i32 %n.vec
  %ind.end17 = getelementptr i16, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i16, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i16, ptr %pDst, i32 %index
  %wide.load = load <8 x i16>, ptr %next.gep, align 2
  %0 = shufflevector <8 x i16> %wide.load, <8 x i16> %wide.load, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
  %1 = shufflevector <8 x i16> %wide.load, <8 x i16> %wide.load, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
  %2 = sext <4 x i16> %0 to <4 x i32>
  %3 = sext <4 x i16> %1 to <4 x i32>
  %wide.load20 = load <8 x i16>, ptr %next.gep18, align 2
  %4 = shufflevector <8 x i16> %wide.load20, <8 x i16> %wide.load20, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
  %5 = shufflevector <8 x i16> %wide.load20, <8 x i16> %wide.load20, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
  %6 = sext <4 x i16> %4 to <4 x i32>
  %7 = sext <4 x i16> %5 to <4 x i32>
  %8 = mul <4 x i32> %6, %2
  %9 = mul <4 x i32> %7, %3
  %10 = ashr <4 x i32> %8, <i32 15, i32 15, i32 15, i32 15>
  %11 = ashr <4 x i32> %9, <i32 15, i32 15, i32 15, i32 15>
  %12 = icmp sgt <4 x i32> %10, <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %13 = icmp sgt <4 x i32> %11, <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %14 = select <4 x i1> %12, <4 x i32> %10, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %15 = select <4 x i1> %13, <4 x i32> %11, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %16 = icmp slt <4 x i32> %14, <i32 32767, i32 32767, i32 32767, i32 32767>
  %17 = icmp slt <4 x i32> %15, <i32 32767, i32 32767, i32 32767, i32 32767>
  %18 = select <4 x i1> %16, <4 x i32> %14, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
  %19 = select <4 x i1> %17, <4 x i32> %15, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
  %20 = shufflevector <4 x i32> %18, <4 x i32> %19, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
  %21 = trunc <8 x i32> %20 to <8 x i16>
  store <8 x i16> %21, ptr %next.gep19, align 2
  %index.next = add i32 %index, 8
  %22 = icmp eq i32 %index.next, %n.vec
  br i1 %22, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body, %for.body.preheader21
  %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ]
  %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ]
  %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ]
  %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ]
  %incdec.ptr = getelementptr inbounds i16, ptr %pSrcA.addr.011, i32 1
  %23 = load i16, ptr %pSrcA.addr.011, align 2
  %conv = sext i16 %23 to i32
  %incdec.ptr1 = getelementptr inbounds i16, ptr %pSrcB.addr.010, i32 1
  %24 = load i16, ptr %pSrcB.addr.010, align 2
  %conv2 = sext i16 %24 to i32
  %mul = mul nsw i32 %conv2, %conv
  %shr = ashr i32 %mul, 15
  %25 = icmp sgt i32 %shr, -32768
  %.val.i = select i1 %25, i32 %shr, i32 -32768
  %26 = icmp slt i32 %.val.i, 32767
  %retval.0.i = select i1 %26, i32 %.val.i, i32 32767
  %conv3 = trunc i32 %retval.0.i to i16
  %incdec.ptr4 = getelementptr inbounds i16, ptr %pDst.addr.09, i32 1
  store i16 %conv3, ptr %pDst.addr.09, align 2
  %inc = add nuw i32 %i.012, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}

define arm_aapcs_vfpcc void @ssatmul_s4t_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_s4t_q15:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, lr}
; CHECK-NEXT:    push {r4, lr}
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    it eq
; CHECK-NEXT:    popeq {r4, pc}
; CHECK-NEXT:  .LBB8_1: @ %vector.ph
; CHECK-NEXT:    add.w r12, r3, #3
; CHECK-NEXT:    mov.w lr, #1
; CHECK-NEXT:    bic r12, r12, #3
; CHECK-NEXT:    adr r4, .LCPI8_0
; CHECK-NEXT:    sub.w r12, r12, #4
; CHECK-NEXT:    vldrw.u32 q0, [r4]
; CHECK-NEXT:    add.w lr, lr, r12, lsr #2
; CHECK-NEXT:    sub.w r12, r3, #1
; CHECK-NEXT:    movs r3, #0
; CHECK-NEXT:    vdup.32 q1, r12
; CHECK-NEXT:  .LBB8_2: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vdup.32 q2, r3
; CHECK-NEXT:    adds r3, #4
; CHECK-NEXT:    vorr q2, q2, q0
; CHECK-NEXT:    vptt.u32 cs, q1, q2
; CHECK-NEXT:    vldrht.s32 q2, [r0], #8
; CHECK-NEXT:    vldrht.s32 q3, [r1], #8
; CHECK-NEXT:    vmul.i32 q2, q3, q2
; CHECK-NEXT:    vqshrnb.s32 q2, q2, #15
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrht.32 q2, [r2], #8
; CHECK-NEXT:    le lr, .LBB8_2
; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r4, pc}
; CHECK-NEXT:    .p2align 4
; CHECK-NEXT:  @ %bb.4:
; CHECK-NEXT:  .LCPI8_0:
; CHECK-NEXT:    .long 0 @ 0x0
; CHECK-NEXT:    .long 1 @ 0x1
; CHECK-NEXT:    .long 2 @ 0x2
; CHECK-NEXT:    .long 3 @ 0x3
entry:
  %cmp8 = icmp eq i32 %N, 0
  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph

vector.ph:                                        ; preds = %entry
  %n.rnd.up = add i32 %N, 3
  %n.vec = and i32 %n.rnd.up, -4
  %trip.count.minus.1 = add i32 %N, -1
  %broadcast.splatinsert20 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
  %broadcast.splat21 = shufflevector <4 x i32> %broadcast.splatinsert20, <4 x i32> undef, <4 x i32> zeroinitializer
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
  %induction = or <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
  %next.gep = getelementptr i16, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i16, ptr %pDst, i32 %index
  %0 = icmp ule <4 x i32> %induction, %broadcast.splat21
  %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %next.gep, i32 2, <4 x i1> %0, <4 x i16> undef)
  %1 = sext <4 x i16> %wide.masked.load to <4 x i32>
  %wide.masked.load22 = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %next.gep18, i32 2, <4 x i1> %0, <4 x i16> undef)
  %2 = sext <4 x i16> %wide.masked.load22 to <4 x i32>
  %3 = mul nsw <4 x i32> %2, %1
  %4 = ashr <4 x i32> %3, <i32 15, i32 15, i32 15, i32 15>
  %5 = icmp sgt <4 x i32> %4, <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %6 = select <4 x i1> %5, <4 x i32> %4, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %7 = icmp slt <4 x i32> %6, <i32 32767, i32 32767, i32 32767, i32 32767>
  %8 = select <4 x i1> %7, <4 x i32> %6, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
  %9 = trunc <4 x i32> %8 to <4 x i16>
  call void @llvm.masked.store.v4i16.p0(<4 x i16> %9, ptr %next.gep19, i32 2, <4 x i1> %0)
  %index.next = add i32 %index, 4
  %10 = icmp eq i32 %index.next, %n.vec
  br i1 %10, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body, %entry
  ret void
}

define arm_aapcs_vfpcc void @ssatmul_8t_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_8t_q15:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r7, lr}
; CHECK-NEXT:    push {r4, r5, r7, lr}
; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    .pad #16
; CHECK-NEXT:    sub sp, #16
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq .LBB9_3
; CHECK-NEXT:  @ %bb.1: @ %vector.ph
; CHECK-NEXT:    adds r4, r3, #7
; CHECK-NEXT:    vmov.i8 q2, #0x0
; CHECK-NEXT:    bic r4, r4, #7
; CHECK-NEXT:    vmov.i8 q3, #0xff
; CHECK-NEXT:    sub.w r12, r4, #8
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    mov r5, sp
; CHECK-NEXT:    add.w lr, r4, r12, lsr #3
; CHECK-NEXT:    adr r4, .LCPI9_0
; CHECK-NEXT:    vldrw.u32 q0, [r4]
; CHECK-NEXT:    adr r4, .LCPI9_1
; CHECK-NEXT:    sub.w r12, r3, #1
; CHECK-NEXT:    vldrw.u32 q4, [r4]
; CHECK-NEXT:    movs r3, #0
; CHECK-NEXT:    vdup.32 q1, r12
; CHECK-NEXT:  .LBB9_2: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vdup.32 q5, r3
; CHECK-NEXT:    adds r3, #8
; CHECK-NEXT:    vorr q6, q5, q0
; CHECK-NEXT:    vorr q5, q5, q4
; CHECK-NEXT:    vcmp.u32 cs, q1, q6
; CHECK-NEXT:    vpsel q6, q3, q2
; CHECK-NEXT:    vcmp.u32 cs, q1, q5
; CHECK-NEXT:    vpsel q5, q3, q2
; CHECK-NEXT:    vstrh.32 q6, [r5, #8]
; CHECK-NEXT:    vstrh.32 q5, [r5]
; CHECK-NEXT:    vldrw.u32 q5, [r5]
; CHECK-NEXT:    vptt.i16 ne, q5, zr
; CHECK-NEXT:    vldrht.u16 q5, [r0], #16
; CHECK-NEXT:    vldrht.u16 q6, [r1], #16
; CHECK-NEXT:    vmullt.s16 q7, q6, q5
; CHECK-NEXT:    vmullb.s16 q5, q6, q5
; CHECK-NEXT:    vqshrnb.s32 q5, q5, #15
; CHECK-NEXT:    vqshrnt.s32 q5, q7, #15
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrht.16 q5, [r2], #16
; CHECK-NEXT:    le lr, .LBB9_2
; CHECK-NEXT:  .LBB9_3: @ %for.cond.cleanup
; CHECK-NEXT:    add sp, #16
; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    pop {r4, r5, r7, pc}
; CHECK-NEXT:    .p2align 4
; CHECK-NEXT:  @ %bb.4:
; CHECK-NEXT:  .LCPI9_0:
; CHECK-NEXT:    .long 4 @ 0x4
; CHECK-NEXT:    .long 5 @ 0x5
; CHECK-NEXT:    .long 6 @ 0x6
; CHECK-NEXT:    .long 7 @ 0x7
; CHECK-NEXT:  .LCPI9_1:
; CHECK-NEXT:    .long 0 @ 0x0
; CHECK-NEXT:    .long 1 @ 0x1
; CHECK-NEXT:    .long 2 @ 0x2
; CHECK-NEXT:    .long 3 @ 0x3
entry:
  %cmp8 = icmp eq i32 %N, 0
  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph

vector.ph:                                        ; preds = %entry
  %n.rnd.up = add i32 %N, 7
  %n.vec = and i32 %n.rnd.up, -8
  %trip.count.minus.1 = add i32 %N, -1
  %broadcast.splatinsert20 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0
  %broadcast.splat21 = shufflevector <8 x i32> %broadcast.splatinsert20, <8 x i32> undef, <8 x i32> zeroinitializer
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
  %induction = or <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  %next.gep = getelementptr i16, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i16, ptr %pDst, i32 %index
  %0 = icmp ule <8 x i32> %induction, %broadcast.splat21
  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %next.gep, i32 2, <8 x i1> %0, <8 x i16> undef)
  %1 = sext <8 x i16> %wide.masked.load to <8 x i32>
  %wide.masked.load22 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %next.gep18, i32 2, <8 x i1> %0, <8 x i16> undef)
  %2 = sext <8 x i16> %wide.masked.load22 to <8 x i32>
  %3 = mul nsw <8 x i32> %2, %1
  %4 = ashr <8 x i32> %3, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
  %5 = icmp sgt <8 x i32> %4, <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %7 = icmp slt <8 x i32> %6, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
  %8 = select <8 x i1> %7, <8 x i32> %6, <8 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
  %9 = trunc <8 x i32> %8 to <8 x i16>
  call void @llvm.masked.store.v8i16.p0(<8 x i16> %9, ptr %next.gep19, i32 2, <8 x i1> %0)
  %index.next = add i32 %index, 8
  %10 = icmp eq i32 %index.next, %n.vec
  br i1 %10, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body, %entry
  ret void
}

define arm_aapcs_vfpcc void @ssatmul_8ti_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_8ti_q15:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r7, lr}
; CHECK-NEXT:    push {r4, r5, r7, lr}
; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    .pad #16
; CHECK-NEXT:    sub sp, #16
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq .LBB10_3
; CHECK-NEXT:  @ %bb.1: @ %vector.ph
; CHECK-NEXT:    adds r4, r3, #7
; CHECK-NEXT:    vmov.i8 q2, #0x0
; CHECK-NEXT:    bic r4, r4, #7
; CHECK-NEXT:    vmov.i8 q3, #0xff
; CHECK-NEXT:    sub.w r12, r4, #8
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    mov r5, sp
; CHECK-NEXT:    add.w lr, r4, r12, lsr #3
; CHECK-NEXT:    adr r4, .LCPI10_0
; CHECK-NEXT:    vldrw.u32 q0, [r4]
; CHECK-NEXT:    adr r4, .LCPI10_1
; CHECK-NEXT:    sub.w r12, r3, #1
; CHECK-NEXT:    vldrw.u32 q4, [r4]
; CHECK-NEXT:    movs r3, #0
; CHECK-NEXT:    vdup.32 q1, r12
; CHECK-NEXT:  .LBB10_2: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vdup.32 q5, r3
; CHECK-NEXT:    adds r3, #8
; CHECK-NEXT:    vorr q6, q5, q0
; CHECK-NEXT:    vorr q5, q5, q4
; CHECK-NEXT:    vcmp.u32 cs, q1, q6
; CHECK-NEXT:    vpsel q6, q3, q2
; CHECK-NEXT:    vcmp.u32 cs, q1, q5
; CHECK-NEXT:    vpsel q5, q3, q2
; CHECK-NEXT:    vstrh.32 q6, [r5, #8]
; CHECK-NEXT:    vstrh.32 q5, [r5]
; CHECK-NEXT:    vldrw.u32 q5, [r5]
; CHECK-NEXT:    vptt.i16 ne, q5, zr
; CHECK-NEXT:    vldrht.u16 q5, [r0], #16
; CHECK-NEXT:    vldrht.u16 q6, [r1], #16
; CHECK-NEXT:    vmullt.s16 q7, q6, q5
; CHECK-NEXT:    vmullb.s16 q5, q6, q5
; CHECK-NEXT:    vqshrnb.s32 q5, q5, #15
; CHECK-NEXT:    vqshrnt.s32 q5, q7, #15
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrht.16 q5, [r2], #16
; CHECK-NEXT:    le lr, .LBB10_2
; CHECK-NEXT:  .LBB10_3: @ %for.cond.cleanup
; CHECK-NEXT:    add sp, #16
; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    pop {r4, r5, r7, pc}
; CHECK-NEXT:    .p2align 4
; CHECK-NEXT:  @ %bb.4:
; CHECK-NEXT:  .LCPI10_0:
; CHECK-NEXT:    .long 4 @ 0x4
; CHECK-NEXT:    .long 5 @ 0x5
; CHECK-NEXT:    .long 6 @ 0x6
; CHECK-NEXT:    .long 7 @ 0x7
; CHECK-NEXT:  .LCPI10_1:
; CHECK-NEXT:    .long 0 @ 0x0
; CHECK-NEXT:    .long 1 @ 0x1
; CHECK-NEXT:    .long 2 @ 0x2
; CHECK-NEXT:    .long 3 @ 0x3
entry:
  %cmp8 = icmp eq i32 %N, 0
  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph

vector.ph:                                        ; preds = %entry
  %n.rnd.up = add i32 %N, 7
  %n.vec = and i32 %n.rnd.up, -8
  %trip.count.minus.1 = add i32 %N, -1
  %broadcast.splatinsert20 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0
  %broadcast.splat21 = shufflevector <8 x i32> %broadcast.splatinsert20, <8 x i32> undef, <8 x i32> zeroinitializer
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
  %induction = or <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  %next.gep = getelementptr i16, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i16, ptr %pDst, i32 %index
  %0 = icmp ule <8 x i32> %induction, %broadcast.splat21
  %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %next.gep, i32 2, <8 x i1> %0, <8 x i16> undef)
  %1 = shufflevector <8 x i16> %wide.masked.load, <8 x i16> %wide.masked.load, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
  %2 = shufflevector <8 x i16> %wide.masked.load, <8 x i16> %wide.masked.load, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
  %3 = sext <4 x i16> %1 to <4 x i32>
  %4 = sext <4 x i16> %2 to <4 x i32>
  %wide.masked.load22 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %next.gep18, i32 2, <8 x i1> %0, <8 x i16> undef)
  %5 = shufflevector <8 x i16> %wide.masked.load22, <8 x i16> %wide.masked.load22, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
  %6 = shufflevector <8 x i16> %wide.masked.load22, <8 x i16> %wide.masked.load22, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
  %7 = sext <4 x i16> %5 to <4 x i32>
  %8 = sext <4 x i16> %6 to <4 x i32>
  %9 = mul <4 x i32> %7, %3
  %10 = mul <4 x i32> %8, %4
  %11 = ashr <4 x i32> %9, <i32 15, i32 15, i32 15, i32 15>
  %12 = ashr <4 x i32> %10, <i32 15, i32 15, i32 15, i32 15>
  %13 = icmp sgt <4 x i32> %11, <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %14 = icmp sgt <4 x i32> %12, <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %15 = select <4 x i1> %13, <4 x i32> %11, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %16 = select <4 x i1> %14, <4 x i32> %12, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
  %17 = icmp slt <4 x i32> %15, <i32 32767, i32 32767, i32 32767, i32 32767>
  %18 = icmp slt <4 x i32> %16, <i32 32767, i32 32767, i32 32767, i32 32767>
  %19 = select <4 x i1> %17, <4 x i32> %15, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
  %20 = select <4 x i1> %18, <4 x i32> %16, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
  %21 = shufflevector <4 x i32> %19, <4 x i32> %20, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
  %22 = trunc <8 x i32> %21 to <8 x i16>
  call void @llvm.masked.store.v8i16.p0(<8 x i16> %22, ptr %next.gep19, i32 2, <8 x i1> %0)
  %index.next = add i32 %index, 8
  %23 = icmp eq i32 %index.next, %n.vec
  br i1 %23, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body, %entry
  ret void
}

define arm_aapcs_vfpcc void @usatmul_4_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: usatmul_4_q15:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, lr}
; CHECK-NEXT:    push {r4, r5, r6, lr}
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq .LBB11_8
; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
; CHECK-NEXT:    cmp r3, #3
; CHECK-NEXT:    bhi .LBB11_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    movs r5, #0
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r6, r1
; CHECK-NEXT:    mov r4, r2
; CHECK-NEXT:    b .LBB11_6
; CHECK-NEXT:  .LBB11_3: @ %vector.ph
; CHECK-NEXT:    bic r5, r3, #3
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    subs r6, r5, #4
; CHECK-NEXT:    add.w r12, r0, r5, lsl #1
; CHECK-NEXT:    add.w lr, r4, r6, lsr #2
; CHECK-NEXT:    add.w r4, r2, r5, lsl #1
; CHECK-NEXT:    add.w r6, r1, r5, lsl #1
; CHECK-NEXT:  .LBB11_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u32 q0, [r0], #8
; CHECK-NEXT:    vldrh.u32 q1, [r1], #8
; CHECK-NEXT:    vmul.i32 q0, q1, q0
; CHECK-NEXT:    vqshrnb.u32 q0, q0, #15
; CHECK-NEXT:    vstrh.32 q0, [r2], #8
; CHECK-NEXT:    le lr, .LBB11_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    cmp r5, r3
; CHECK-NEXT:    it eq
; CHECK-NEXT:    popeq {r4, r5, r6, pc}
; CHECK-NEXT:  .LBB11_6: @ %for.body.preheader21
; CHECK-NEXT:    sub.w lr, r3, r5
; CHECK-NEXT:    movw r0, #65535
; CHECK-NEXT:  .LBB11_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldrh r1, [r12], #2
; CHECK-NEXT:    ldrh r2, [r6], #2
; CHECK-NEXT:    muls r1, r2, r1
; CHECK-NEXT:    lsrs r2, r1, #15
; CHECK-NEXT:    cmp r2, r0
; CHECK-NEXT:    movw r2, #65535
; CHECK-NEXT:    it lo
; CHECK-NEXT:    lsrlo r2, r1, #15
; CHECK-NEXT:    strh r2, [r4], #2
; CHECK-NEXT:    le lr, .LBB11_7
; CHECK-NEXT:  .LBB11_8: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r4, r5, r6, pc}
entry:
  %cmp8 = icmp eq i32 %N, 0
  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry
  %min.iters.check = icmp ult i32 %N, 4
  br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph

for.body.preheader21:                             ; preds = %middle.block, %for.body.preheader
  %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
  %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ]
  %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ]
  %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ]
  br label %for.body

vector.ph:                                        ; preds = %for.body.preheader
  %n.vec = and i32 %N, -4
  %ind.end = getelementptr i16, ptr %pSrcA, i32 %n.vec
  %ind.end15 = getelementptr i16, ptr %pSrcB, i32 %n.vec
  %ind.end17 = getelementptr i16, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i16, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i16, ptr %pDst, i32 %index
  %wide.load = load <4 x i16>, ptr %next.gep, align 2
  %0 = zext <4 x i16> %wide.load to <4 x i32>
  %wide.load20 = load <4 x i16>, ptr %next.gep18, align 2
  %1 = zext <4 x i16> %wide.load20 to <4 x i32>
  %2 = mul nuw <4 x i32> %1, %0
  %3 = lshr <4 x i32> %2, <i32 15, i32 15, i32 15, i32 15>
  %4 = icmp ult <4 x i32> %3, <i32 65535, i32 65535, i32 65535, i32 65535>
  %5 = select <4 x i1> %4, <4 x i32> %3, <4 x i32> <i32 65535, i32 65535, i32 65535, i32 65535>
  %6 = trunc <4 x i32> %5 to <4 x i16>
  store <4 x i16> %6, ptr %next.gep19, align 2
  %index.next = add i32 %index, 4
  %7 = icmp eq i32 %index.next, %n.vec
  br i1 %7, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body.preheader21, %for.body
  %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ]
  %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ]
  %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ]
  %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ]
  %incdec.ptr = getelementptr inbounds i16, ptr %pSrcA.addr.011, i32 1
  %8 = load i16, ptr %pSrcA.addr.011, align 2
  %conv = zext i16 %8 to i32
  %incdec.ptr1 = getelementptr inbounds i16, ptr %pSrcB.addr.010, i32 1
  %9 = load i16, ptr %pSrcB.addr.010, align 2
  %conv2 = zext i16 %9 to i32
  %mul = mul nuw i32 %conv2, %conv
  %shr = lshr i32 %mul, 15
  %10 = icmp ult i32 %shr, 65535
  %retval.0.i = select i1 %10, i32 %shr, i32 65535
  %conv3 = trunc i32 %retval.0.i to i16
  %incdec.ptr4 = getelementptr inbounds i16, ptr %pDst.addr.09, i32 1
  store i16 %conv3, ptr %pDst.addr.09, align 2
  %inc = add nuw i32 %i.012, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}

define arm_aapcs_vfpcc void @usatmul_8_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: usatmul_8_q15:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, lr}
; CHECK-NEXT:    push {r4, r5, r6, lr}
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq .LBB12_8
; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
; CHECK-NEXT:    cmp r3, #7
; CHECK-NEXT:    bhi .LBB12_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    movs r5, #0
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r6, r1
; CHECK-NEXT:    mov r4, r2
; CHECK-NEXT:    b .LBB12_6
; CHECK-NEXT:  .LBB12_3: @ %vector.ph
; CHECK-NEXT:    bic r5, r3, #7
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    sub.w r6, r5, #8
; CHECK-NEXT:    add.w r12, r0, r5, lsl #1
; CHECK-NEXT:    add.w lr, r4, r6, lsr #3
; CHECK-NEXT:    add.w r4, r2, r5, lsl #1
; CHECK-NEXT:    add.w r6, r1, r5, lsl #1
; CHECK-NEXT:  .LBB12_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
; CHECK-NEXT:    vmullt.u16 q2, q1, q0
; CHECK-NEXT:    vmullb.u16 q0, q1, q0
; CHECK-NEXT:    vqshrnb.u32 q0, q0, #15
; CHECK-NEXT:    vqshrnt.u32 q0, q2, #15
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB12_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    cmp r5, r3
; CHECK-NEXT:    it eq
; CHECK-NEXT:    popeq {r4, r5, r6, pc}
; CHECK-NEXT:  .LBB12_6: @ %for.body.preheader21
; CHECK-NEXT:    sub.w lr, r3, r5
; CHECK-NEXT:    movw r0, #65535
; CHECK-NEXT:  .LBB12_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldrh r1, [r12], #2
; CHECK-NEXT:    ldrh r2, [r6], #2
; CHECK-NEXT:    muls r1, r2, r1
; CHECK-NEXT:    lsrs r2, r1, #15
; CHECK-NEXT:    cmp r2, r0
; CHECK-NEXT:    movw r2, #65535
; CHECK-NEXT:    it lo
; CHECK-NEXT:    lsrlo r2, r1, #15
; CHECK-NEXT:    strh r2, [r4], #2
; CHECK-NEXT:    le lr, .LBB12_7
; CHECK-NEXT:  .LBB12_8: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r4, r5, r6, pc}
entry:
  %cmp8 = icmp eq i32 %N, 0
  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry
  %min.iters.check = icmp ult i32 %N, 8
  br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph

for.body.preheader21:                             ; preds = %middle.block, %for.body.preheader
  %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
  %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ]
  %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ]
  %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ]
  br label %for.body

vector.ph:                                        ; preds = %for.body.preheader
  %n.vec = and i32 %N, -8
  %ind.end = getelementptr i16, ptr %pSrcA, i32 %n.vec
  %ind.end15 = getelementptr i16, ptr %pSrcB, i32 %n.vec
  %ind.end17 = getelementptr i16, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i16, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i16, ptr %pDst, i32 %index
  %wide.load = load <8 x i16>, ptr %next.gep, align 2
  %0 = zext <8 x i16> %wide.load to <8 x i32>
  %wide.load20 = load <8 x i16>, ptr %next.gep18, align 2
  %1 = zext <8 x i16> %wide.load20 to <8 x i32>
  %2 = mul nuw <8 x i32> %1, %0
  %3 = lshr <8 x i32> %2, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
  %4 = icmp ult <8 x i32> %3, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
  %5 = select <8 x i1> %4, <8 x i32> %3, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
  %6 = trunc <8 x i32> %5 to <8 x i16>
  store <8 x i16> %6, ptr %next.gep19, align 2
  %index.next = add i32 %index, 8
  %7 = icmp eq i32 %index.next, %n.vec
  br i1 %7, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body.preheader21, %for.body
  %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ]
  %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ]
  %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ]
  %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ]
  %incdec.ptr = getelementptr inbounds i16, ptr %pSrcA.addr.011, i32 1
  %8 = load i16, ptr %pSrcA.addr.011, align 2
  %conv = zext i16 %8 to i32
  %incdec.ptr1 = getelementptr inbounds i16, ptr %pSrcB.addr.010, i32 1
  %9 = load i16, ptr %pSrcB.addr.010, align 2
  %conv2 = zext i16 %9 to i32
  %mul = mul nuw i32 %conv2, %conv
  %shr = lshr i32 %mul, 15
  %10 = icmp ult i32 %shr, 65535
  %retval.0.i = select i1 %10, i32 %shr, i32 65535
  %conv3 = trunc i32 %retval.0.i to i16
  %incdec.ptr4 = getelementptr inbounds i16, ptr %pDst.addr.09, i32 1
  store i16 %conv3, ptr %pDst.addr.09, align 2
  %inc = add nuw i32 %i.012, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}


; i8

define arm_aapcs_vfpcc void @ssatmul_4_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_4_q7:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, lr}
; CHECK-NEXT:    push {r4, r5, r6, lr}
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq .LBB13_8
; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
; CHECK-NEXT:    cmp r3, #3
; CHECK-NEXT:    bhi .LBB13_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    movs r5, #0
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r6, r1
; CHECK-NEXT:    mov r4, r2
; CHECK-NEXT:    b .LBB13_6
; CHECK-NEXT:  .LBB13_3: @ %vector.ph
; CHECK-NEXT:    bic r5, r3, #3
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    subs r6, r5, #4
; CHECK-NEXT:    add.w r12, r0, r5
; CHECK-NEXT:    vmvn.i32 q0, #0x7f
; CHECK-NEXT:    vmov.i32 q1, #0x7f
; CHECK-NEXT:    add.w lr, r4, r6, lsr #2
; CHECK-NEXT:    adds r4, r2, r5
; CHECK-NEXT:    adds r6, r1, r5
; CHECK-NEXT:  .LBB13_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.s32 q2, [r0], #4
; CHECK-NEXT:    vldrb.s32 q3, [r1], #4
; CHECK-NEXT:    vmul.i32 q2, q3, q2
; CHECK-NEXT:    vshr.s32 q2, q2, #7
; CHECK-NEXT:    vmax.s32 q2, q2, q0
; CHECK-NEXT:    vmin.s32 q2, q2, q1
; CHECK-NEXT:    vstrb.32 q2, [r2], #4
; CHECK-NEXT:    le lr, .LBB13_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    cmp r5, r3
; CHECK-NEXT:    it eq
; CHECK-NEXT:    popeq {r4, r5, r6, pc}
; CHECK-NEXT:  .LBB13_6: @ %for.body.preheader21
; CHECK-NEXT:    sub.w lr, r3, r5
; CHECK-NEXT:  .LBB13_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldrsb r0, [r12], #1
; CHECK-NEXT:    ldrsb r1, [r6], #1
; CHECK-NEXT:    muls r0, r1, r0
; CHECK-NEXT:    ssat r0, #8, r0, asr #7
; CHECK-NEXT:    strb r0, [r4], #1
; CHECK-NEXT:    le lr, .LBB13_7
; CHECK-NEXT:  .LBB13_8: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r4, r5, r6, pc}
entry:
  %cmp8 = icmp eq i32 %N, 0
  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry
  %min.iters.check = icmp ult i32 %N, 4
  br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph

for.body.preheader21:                             ; preds = %middle.block, %for.body.preheader
  %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
  %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ]
  %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ]
  %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ]
  br label %for.body

vector.ph:                                        ; preds = %for.body.preheader
  %n.vec = and i32 %N, -4
  %ind.end = getelementptr i8, ptr %pSrcA, i32 %n.vec
  %ind.end15 = getelementptr i8, ptr %pSrcB, i32 %n.vec
  %ind.end17 = getelementptr i8, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i8, ptr %pSrcA, i32 %index
  %next.gep18 = getelementptr i8, ptr %pSrcB, i32 %index
  %next.gep19 = getelementptr i8, ptr %pDst, i32 %index
  %wide.load = load <4 x i8>, ptr %next.gep, align 1
  %0 = sext <4 x i8> %wide.load to <4 x i32>
  %wide.load20 = load <4 x i8>, ptr %next.gep18, align 1
  %1 = sext <4 x i8> %wide.load20 to <4 x i32>
  %2 = mul nsw <4 x i32> %1, %0
  %3 = ashr <4 x i32> %2, <i32 7, i32 7, i32 7, i32 7>
  %4 = icmp sgt <4 x i32> %3, <i32 -128, i32 -128, i32 -128, i32 -128>
  %5 = select <4 x i1> %4, <4 x i32> %3, <4 x i32> <i32 -128, i32 -128, i32 -128, i32 -128>
  %6 = icmp slt <4 x i32> %5, <i32 127, i32 127, i32 127, i32 127>
  %7 = select <4 x i1> %6, <4 x i32> %5, <4 x i32> <i32 127, i32 127, i32 127, i32 127>
  %8 = trunc <4 x i32> %7 to <4 x i8>
  store <4 x i8> %8, ptr %next.gep19, align 1
  %index.next = add i32 %index, 4
  %9 = icmp eq i32 %index.next, %n.vec
  br i1 %9, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body.preheader21, %for.body
  %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ]
  %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ]
  %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ]
  %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ]
  %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.011, i32 1
  %10 = load i8, ptr %pSrcA.addr.011, align 1
  %conv = sext i8 %10 to i32
  %incdec.ptr1 = getelementptr inbounds i8, ptr %pSrcB.addr.010, i32 1
  %11 = load i8, ptr %pSrcB.addr.010, align 1
  %conv2 = sext i8 %11 to i32
  %mul = mul nsw i32 %conv2, %conv
  %shr = ashr i32 %mul, 7
  %12 = icmp sgt i32 %shr, -128
  %.val.i = select i1 %12, i32 %shr, i32 -128
  %13 = icmp slt i32 %.val.i, 127
  %retval.0.i = select i1 %13, i32 %.val.i, i32 127
  %conv3 = trunc i32 %retval.0.i to i8
  %incdec.ptr4 = getelementptr inbounds i8, ptr %pDst.addr.09, i32 1
  store i8 %conv3, ptr %pDst.addr.09, align 1
  %inc = add nuw i32 %i.012, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}

define arm_aapcs_vfpcc void @ssatmul_8_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_8_q7:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, lr}
; CHECK-NEXT:    push {r4, r5, r6, lr}
; CHECK-NEXT:    cbz r3, .LBB14_8
; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
; CHECK-NEXT:    cmp r3, #7
; CHECK-NEXT:    bhi .LBB14_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    movs r5, #0
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r6, r1
; CHECK-NEXT:    mov r4, r2
; CHECK-NEXT:    b .LBB14_6
; CHECK-NEXT:  .LBB14_3: @ %vector.ph
; CHECK-NEXT:    bic r5, r3, #7
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    sub.w r6, r5, #8
; CHECK-NEXT:    add.w r12, r0, r5
; CHECK-NEXT:    add.w lr, r4, r6, lsr #3
; CHECK-NEXT:    adds r4, r2, r5
; CHECK-NEXT:    adds r6, r1, r5
; CHECK-NEXT:  .LBB14_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.s16 q0, [r0], #8
; CHECK-NEXT:    vldrb.s16 q1, [r1], #8
; CHECK-NEXT:    vmul.i16 q0, q1, q0
; CHECK-NEXT:    vqshrnb.s16 q0, q0, #7
; CHECK-NEXT:    vstrb.16 q0, [r2], #8
; CHECK-NEXT:    le lr, .LBB14_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    cmp r5, r3
; CHECK-NEXT:    it eq
; CHECK-NEXT:    popeq {r4, r5, r6, pc}
; CHECK-NEXT:  .LBB14_6: @ %for.body.preheader23
; CHECK-NEXT:    sub.w lr, r3, r5
; CHECK-NEXT:  .LBB14_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldrsb r0, [r12], #1
; CHECK-NEXT:    ldrsb r1, [r6], #1
; CHECK-NEXT:    muls r0, r1, r0
; CHECK-NEXT:    ssat r0, #8, r0, asr #7
; CHECK-NEXT:    strb r0, [r4], #1
; CHECK-NEXT:    le lr, .LBB14_7
; CHECK-NEXT:  .LBB14_8: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r4, r5, r6, pc}
entry:
  %cmp10 = icmp eq i32 %N, 0
  br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry
  %min.iters.check = icmp ult i32 %N, 8
  br i1 %min.iters.check, label %for.body.preheader23, label %vector.ph

for.body.preheader23:                             ; preds = %middle.block, %for.body.preheader
  %i.014.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
  %pSrcA.addr.013.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ]
  %pSrcB.addr.012.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end17, %middle.block ]
  %pDst.addr.011.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end19, %middle.block ]
  br label %for.body

vector.ph:                                        ; preds = %for.body.preheader
  %n.vec = and i32 %N, -8
  %ind.end = getelementptr i8, ptr %pSrcA, i32 %n.vec
  %ind.end17 = getelementptr i8, ptr %pSrcB, i32 %n.vec
  %ind.end19 = getelementptr i8, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i8, ptr %pSrcA, i32 %index
  %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index
  %next.gep21 = getelementptr i8, ptr %pDst, i32 %index
  %wide.load = load <8 x i8>, ptr %next.gep, align 1
  %0 = sext <8 x i8> %wide.load to <8 x i16>
  %wide.load22 = load <8 x i8>, ptr %next.gep20, align 1
  %1 = sext <8 x i8> %wide.load22 to <8 x i16>
  %2 = mul nsw <8 x i16> %1, %0
  %3 = ashr <8 x i16> %2, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
  %4 = icmp sgt <8 x i16> %3, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %5 = select <8 x i1> %4, <8 x i16> %3, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %6 = icmp slt <8 x i16> %5, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %7 = select <8 x i1> %6, <8 x i16> %5, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %8 = trunc <8 x i16> %7 to <8 x i8>
  store <8 x i8> %8, ptr %next.gep21, align 1
  %index.next = add i32 %index, 8
  %9 = icmp eq i32 %index.next, %n.vec
  br i1 %9, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader23

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body.preheader23, %for.body
  %i.014 = phi i32 [ %inc, %for.body ], [ %i.014.ph, %for.body.preheader23 ]
  %pSrcA.addr.013 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.013.ph, %for.body.preheader23 ]
  %pSrcB.addr.012 = phi ptr [ %incdec.ptr2, %for.body ], [ %pSrcB.addr.012.ph, %for.body.preheader23 ]
  %pDst.addr.011 = phi ptr [ %incdec.ptr6, %for.body ], [ %pDst.addr.011.ph, %for.body.preheader23 ]
  %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.013, i32 1
  %10 = load i8, ptr %pSrcA.addr.013, align 1
  %conv1 = sext i8 %10 to i16
  %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.012, i32 1
  %11 = load i8, ptr %pSrcB.addr.012, align 1
  %conv3 = sext i8 %11 to i16
  %mul = mul nsw i16 %conv3, %conv1
  %shr = ashr i16 %mul, 7
  %12 = icmp sgt i16 %shr, -128
  %.val.i = select i1 %12, i16 %shr, i16 -128
  %13 = icmp slt i16 %.val.i, 127
  %retval.0.i = select i1 %13, i16 %.val.i, i16 127
  %conv5 = trunc i16 %retval.0.i to i8
  %incdec.ptr6 = getelementptr inbounds i8, ptr %pDst.addr.011, i32 1
  store i8 %conv5, ptr %pDst.addr.011, align 1
  %inc = add nuw i32 %i.014, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}

define arm_aapcs_vfpcc void @ssatmul_16_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_16_q7:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, lr}
; CHECK-NEXT:    push {r4, r5, r6, lr}
; CHECK-NEXT:    cbz r3, .LBB15_8
; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
; CHECK-NEXT:    cmp r3, #15
; CHECK-NEXT:    bhi .LBB15_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    movs r5, #0
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r6, r1
; CHECK-NEXT:    mov r4, r2
; CHECK-NEXT:    b .LBB15_6
; CHECK-NEXT:  .LBB15_3: @ %vector.ph
; CHECK-NEXT:    bic r5, r3, #15
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    sub.w r6, r5, #16
; CHECK-NEXT:    add.w r12, r0, r5
; CHECK-NEXT:    add.w lr, r4, r6, lsr #4
; CHECK-NEXT:    adds r4, r2, r5
; CHECK-NEXT:    adds r6, r1, r5
; CHECK-NEXT:  .LBB15_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
; CHECK-NEXT:    vmullt.s8 q2, q1, q0
; CHECK-NEXT:    vmullb.s8 q0, q1, q0
; CHECK-NEXT:    vqshrnb.s16 q0, q0, #7
; CHECK-NEXT:    vqshrnt.s16 q0, q2, #7
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB15_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    cmp r5, r3
; CHECK-NEXT:    it eq
; CHECK-NEXT:    popeq {r4, r5, r6, pc}
; CHECK-NEXT:  .LBB15_6: @ %for.body.preheader23
; CHECK-NEXT:    sub.w lr, r3, r5
; CHECK-NEXT:  .LBB15_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldrsb r0, [r12], #1
; CHECK-NEXT:    ldrsb r1, [r6], #1
; CHECK-NEXT:    muls r0, r1, r0
; CHECK-NEXT:    ssat r0, #8, r0, asr #7
; CHECK-NEXT:    strb r0, [r4], #1
; CHECK-NEXT:    le lr, .LBB15_7
; CHECK-NEXT:  .LBB15_8: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r4, r5, r6, pc}
entry:
  %cmp10 = icmp eq i32 %N, 0
  br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry
  %min.iters.check = icmp ult i32 %N, 16
  br i1 %min.iters.check, label %for.body.preheader23, label %vector.ph

for.body.preheader23:                             ; preds = %middle.block, %for.body.preheader
  %i.014.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
  %pSrcA.addr.013.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ]
  %pSrcB.addr.012.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end17, %middle.block ]
  %pDst.addr.011.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end19, %middle.block ]
  br label %for.body

vector.ph:                                        ; preds = %for.body.preheader
  %n.vec = and i32 %N, -16
  %ind.end = getelementptr i8, ptr %pSrcA, i32 %n.vec
  %ind.end17 = getelementptr i8, ptr %pSrcB, i32 %n.vec
  %ind.end19 = getelementptr i8, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i8, ptr %pSrcA, i32 %index
  %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index
  %next.gep21 = getelementptr i8, ptr %pDst, i32 %index
  %wide.load = load <16 x i8>, ptr %next.gep, align 1
  %0 = sext <16 x i8> %wide.load to <16 x i16>
  %wide.load22 = load <16 x i8>, ptr %next.gep20, align 1
  %1 = sext <16 x i8> %wide.load22 to <16 x i16>
  %2 = mul nsw <16 x i16> %1, %0
  %3 = ashr <16 x i16> %2, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
  %4 = icmp sgt <16 x i16> %3, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %5 = select <16 x i1> %4, <16 x i16> %3, <16 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %6 = icmp slt <16 x i16> %5, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %7 = select <16 x i1> %6, <16 x i16> %5, <16 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %8 = trunc <16 x i16> %7 to <16 x i8>
  store <16 x i8> %8, ptr %next.gep21, align 1
  %index.next = add i32 %index, 16
  %9 = icmp eq i32 %index.next, %n.vec
  br i1 %9, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader23

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body.preheader23, %for.body
  %i.014 = phi i32 [ %inc, %for.body ], [ %i.014.ph, %for.body.preheader23 ]
  %pSrcA.addr.013 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.013.ph, %for.body.preheader23 ]
  %pSrcB.addr.012 = phi ptr [ %incdec.ptr2, %for.body ], [ %pSrcB.addr.012.ph, %for.body.preheader23 ]
  %pDst.addr.011 = phi ptr [ %incdec.ptr6, %for.body ], [ %pDst.addr.011.ph, %for.body.preheader23 ]
  %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.013, i32 1
  %10 = load i8, ptr %pSrcA.addr.013, align 1
  %conv1 = sext i8 %10 to i16
  %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.012, i32 1
  %11 = load i8, ptr %pSrcB.addr.012, align 1
  %conv3 = sext i8 %11 to i16
  %mul = mul nsw i16 %conv3, %conv1
  %shr = ashr i16 %mul, 7
  %12 = icmp sgt i16 %shr, -128
  %.val.i = select i1 %12, i16 %shr, i16 -128
  %13 = icmp slt i16 %.val.i, 127
  %retval.0.i = select i1 %13, i16 %.val.i, i16 127
  %conv5 = trunc i16 %retval.0.i to i8
  %incdec.ptr6 = getelementptr inbounds i8, ptr %pDst.addr.011, i32 1
  store i8 %conv5, ptr %pDst.addr.011, align 1
  %inc = add nuw i32 %i.014, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}

define arm_aapcs_vfpcc void @ssatmul_16i_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_16i_q7:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, lr}
; CHECK-NEXT:    push {r4, r5, r6, lr}
; CHECK-NEXT:    cbz r3, .LBB16_8
; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
; CHECK-NEXT:    cmp r3, #15
; CHECK-NEXT:    bhi .LBB16_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    movs r5, #0
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r6, r1
; CHECK-NEXT:    mov r4, r2
; CHECK-NEXT:    b .LBB16_6
; CHECK-NEXT:  .LBB16_3: @ %vector.ph
; CHECK-NEXT:    bic r5, r3, #15
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    sub.w r6, r5, #16
; CHECK-NEXT:    add.w r12, r0, r5
; CHECK-NEXT:    add.w lr, r4, r6, lsr #4
; CHECK-NEXT:    adds r4, r2, r5
; CHECK-NEXT:    adds r6, r1, r5
; CHECK-NEXT:  .LBB16_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
; CHECK-NEXT:    vmullt.s8 q2, q1, q0
; CHECK-NEXT:    vmullb.s8 q0, q1, q0
; CHECK-NEXT:    vqshrnb.s16 q0, q0, #7
; CHECK-NEXT:    vqshrnt.s16 q0, q2, #7
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB16_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    cmp r5, r3
; CHECK-NEXT:    it eq
; CHECK-NEXT:    popeq {r4, r5, r6, pc}
; CHECK-NEXT:  .LBB16_6: @ %for.body.preheader23
; CHECK-NEXT:    sub.w lr, r3, r5
; CHECK-NEXT:  .LBB16_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldrsb r0, [r12], #1
; CHECK-NEXT:    ldrsb r1, [r6], #1
; CHECK-NEXT:    muls r0, r1, r0
; CHECK-NEXT:    ssat r0, #8, r0, asr #7
; CHECK-NEXT:    strb r0, [r4], #1
; CHECK-NEXT:    le lr, .LBB16_7
; CHECK-NEXT:  .LBB16_8: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r4, r5, r6, pc}
entry:
  %cmp10 = icmp eq i32 %N, 0
  br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry
  %min.iters.check = icmp ult i32 %N, 16
  br i1 %min.iters.check, label %for.body.preheader23, label %vector.ph

for.body.preheader23:                             ; preds = %middle.block, %for.body.preheader
  %i.014.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
  %pSrcA.addr.013.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ]
  %pSrcB.addr.012.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end17, %middle.block ]
  %pDst.addr.011.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end19, %middle.block ]
  br label %for.body

vector.ph:                                        ; preds = %for.body.preheader
  %n.vec = and i32 %N, -16
  %ind.end = getelementptr i8, ptr %pSrcA, i32 %n.vec
  %ind.end17 = getelementptr i8, ptr %pSrcB, i32 %n.vec
  %ind.end19 = getelementptr i8, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i8, ptr %pSrcA, i32 %index
  %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index
  %next.gep21 = getelementptr i8, ptr %pDst, i32 %index
  %wide.load = load <16 x i8>, ptr %next.gep, align 1
  %0 = shufflevector <16 x i8> %wide.load, <16 x i8> %wide.load, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
  %1 = shufflevector <16 x i8> %wide.load, <16 x i8> %wide.load, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
  %2 = sext <8 x i8> %0 to <8 x i16>
  %3 = sext <8 x i8> %1 to <8 x i16>
  %wide.load22 = load <16 x i8>, ptr %next.gep20, align 1
  %4 = shufflevector <16 x i8> %wide.load22, <16 x i8> %wide.load22, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
  %5 = shufflevector <16 x i8> %wide.load22, <16 x i8> %wide.load22, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
  %6 = sext <8 x i8> %4 to <8 x i16>
  %7 = sext <8 x i8> %5 to <8 x i16>
  %8 = mul <8 x i16> %6, %2
  %9 = mul <8 x i16> %7, %3
  %10 = ashr <8 x i16> %8, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
  %11 = ashr <8 x i16> %9, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
  %12 = icmp sgt <8 x i16> %10, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %13 = icmp sgt <8 x i16> %11, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %14 = select <8 x i1> %12, <8 x i16> %10, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %15 = select <8 x i1> %13, <8 x i16> %11, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %16 = icmp slt <8 x i16> %14, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %17 = icmp slt <8 x i16> %15, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %18 = select <8 x i1> %16, <8 x i16> %14, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %19 = select <8 x i1> %17, <8 x i16> %15, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %20 = shufflevector <8 x i16> %18, <8 x i16> %19, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
  %21 = trunc <16 x i16> %20 to <16 x i8>
  store <16 x i8> %21, ptr %next.gep21, align 1
  %index.next = add i32 %index, 16
  %22 = icmp eq i32 %index.next, %n.vec
  br i1 %22, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader23

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body, %for.body.preheader23
  %i.014 = phi i32 [ %inc, %for.body ], [ %i.014.ph, %for.body.preheader23 ]
  %pSrcA.addr.013 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.013.ph, %for.body.preheader23 ]
  %pSrcB.addr.012 = phi ptr [ %incdec.ptr2, %for.body ], [ %pSrcB.addr.012.ph, %for.body.preheader23 ]
  %pDst.addr.011 = phi ptr [ %incdec.ptr6, %for.body ], [ %pDst.addr.011.ph, %for.body.preheader23 ]
  %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.013, i32 1
  %23 = load i8, ptr %pSrcA.addr.013, align 1
  %conv1 = sext i8 %23 to i16
  %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.012, i32 1
  %24 = load i8, ptr %pSrcB.addr.012, align 1
  %conv3 = sext i8 %24 to i16
  %mul = mul nsw i16 %conv3, %conv1
  %shr = ashr i16 %mul, 7
  %25 = icmp sgt i16 %shr, -128
  %.val.i = select i1 %25, i16 %shr, i16 -128
  %26 = icmp slt i16 %.val.i, 127
  %retval.0.i = select i1 %26, i16 %.val.i, i16 127
  %conv5 = trunc i16 %retval.0.i to i8
  %incdec.ptr6 = getelementptr inbounds i8, ptr %pDst.addr.011, i32 1
  store i8 %conv5, ptr %pDst.addr.011, align 1
  %inc = add nuw i32 %i.014, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}

define arm_aapcs_vfpcc void @ssatmul_8t_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_8t_q7:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r7, lr}
; CHECK-NEXT:    push {r4, r5, r7, lr}
; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
; CHECK-NEXT:    .pad #16
; CHECK-NEXT:    sub sp, #16
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq .LBB17_3
; CHECK-NEXT:  @ %bb.1: @ %vector.ph
; CHECK-NEXT:    adds r4, r3, #7
; CHECK-NEXT:    vmov.i8 q2, #0x0
; CHECK-NEXT:    bic r4, r4, #7
; CHECK-NEXT:    vmov.i8 q3, #0xff
; CHECK-NEXT:    sub.w r12, r4, #8
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    mov r5, sp
; CHECK-NEXT:    add.w lr, r4, r12, lsr #3
; CHECK-NEXT:    adr r4, .LCPI17_0
; CHECK-NEXT:    vldrw.u32 q0, [r4]
; CHECK-NEXT:    adr r4, .LCPI17_1
; CHECK-NEXT:    sub.w r12, r3, #1
; CHECK-NEXT:    vldrw.u32 q4, [r4]
; CHECK-NEXT:    movs r3, #0
; CHECK-NEXT:    vdup.32 q1, r12
; CHECK-NEXT:  .LBB17_2: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vdup.32 q5, r3
; CHECK-NEXT:    adds r3, #8
; CHECK-NEXT:    vorr q6, q5, q0
; CHECK-NEXT:    vorr q5, q5, q4
; CHECK-NEXT:    vcmp.u32 cs, q1, q6
; CHECK-NEXT:    vpsel q6, q3, q2
; CHECK-NEXT:    vcmp.u32 cs, q1, q5
; CHECK-NEXT:    vpsel q5, q3, q2
; CHECK-NEXT:    vstrh.32 q6, [r5, #8]
; CHECK-NEXT:    vstrh.32 q5, [r5]
; CHECK-NEXT:    vldrw.u32 q5, [r5]
; CHECK-NEXT:    vptt.i16 ne, q5, zr
; CHECK-NEXT:    vldrbt.s16 q5, [r0], #8
; CHECK-NEXT:    vldrbt.s16 q6, [r1], #8
; CHECK-NEXT:    vmul.i16 q5, q6, q5
; CHECK-NEXT:    vqshrnb.s16 q5, q5, #7
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrbt.16 q5, [r2], #8
; CHECK-NEXT:    le lr, .LBB17_2
; CHECK-NEXT:  .LBB17_3: @ %for.cond.cleanup
; CHECK-NEXT:    add sp, #16
; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
; CHECK-NEXT:    pop {r4, r5, r7, pc}
; CHECK-NEXT:    .p2align 4
; CHECK-NEXT:  @ %bb.4:
; CHECK-NEXT:  .LCPI17_0:
; CHECK-NEXT:    .long 4 @ 0x4
; CHECK-NEXT:    .long 5 @ 0x5
; CHECK-NEXT:    .long 6 @ 0x6
; CHECK-NEXT:    .long 7 @ 0x7
; CHECK-NEXT:  .LCPI17_1:
; CHECK-NEXT:    .long 0 @ 0x0
; CHECK-NEXT:    .long 1 @ 0x1
; CHECK-NEXT:    .long 2 @ 0x2
; CHECK-NEXT:    .long 3 @ 0x3
entry:
  %cmp10 = icmp eq i32 %N, 0
  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph

vector.ph:                                        ; preds = %entry
  %n.rnd.up = add i32 %N, 7
  %n.vec = and i32 %n.rnd.up, -8
  %trip.count.minus.1 = add i32 %N, -1
  %broadcast.splatinsert22 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0
  %broadcast.splat23 = shufflevector <8 x i32> %broadcast.splatinsert22, <8 x i32> undef, <8 x i32> zeroinitializer
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0
  %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer
  %induction = or <8 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  %next.gep = getelementptr i8, ptr %pSrcA, i32 %index
  %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index
  %next.gep21 = getelementptr i8, ptr %pDst, i32 %index
  %0 = icmp ule <8 x i32> %induction, %broadcast.splat23
  %wide.masked.load = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr %next.gep, i32 1, <8 x i1> %0, <8 x i8> undef)
  %1 = sext <8 x i8> %wide.masked.load to <8 x i16>
  %wide.masked.load24 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr %next.gep20, i32 1, <8 x i1> %0, <8 x i8> undef)
  %2 = sext <8 x i8> %wide.masked.load24 to <8 x i16>
  %3 = mul nsw <8 x i16> %2, %1
  %4 = ashr <8 x i16> %3, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
  %5 = icmp sgt <8 x i16> %4, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %6 = select <8 x i1> %5, <8 x i16> %4, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %7 = icmp slt <8 x i16> %6, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %8 = select <8 x i1> %7, <8 x i16> %6, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %9 = trunc <8 x i16> %8 to <8 x i8>
  call void @llvm.masked.store.v8i8.p0(<8 x i8> %9, ptr %next.gep21, i32 1, <8 x i1> %0)
  %index.next = add i32 %index, 8
  %10 = icmp eq i32 %index.next, %n.vec
  br i1 %10, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body, %entry
  ret void
}

define arm_aapcs_vfpcc void @ssatmul_16t_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_16t_q7:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, lr}
; CHECK-NEXT:    push {r4, r5, r6, lr}
; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    .pad #80
; CHECK-NEXT:    sub sp, #80
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq .LBB18_3
; CHECK-NEXT:  @ %bb.1: @ %vector.ph
; CHECK-NEXT:    add.w r6, r3, #15
; CHECK-NEXT:    movs r5, #1
; CHECK-NEXT:    bic r6, r6, #15
; CHECK-NEXT:    add r4, sp, #48
; CHECK-NEXT:    subs r6, #16
; CHECK-NEXT:    vmov.i8 q2, #0x0
; CHECK-NEXT:    vmov.i8 q3, #0xff
; CHECK-NEXT:    add.w lr, r5, r6, lsr #4
; CHECK-NEXT:    adr r5, .LCPI18_0
; CHECK-NEXT:    subs r6, r3, #1
; CHECK-NEXT:    vldrw.u32 q0, [r5]
; CHECK-NEXT:    vdup.32 q1, r6
; CHECK-NEXT:    adr r6, .LCPI18_1
; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
; CHECK-NEXT:    vldrw.u32 q0, [r6]
; CHECK-NEXT:    adr r6, .LCPI18_2
; CHECK-NEXT:    vldrw.u32 q5, [r6]
; CHECK-NEXT:    adr r6, .LCPI18_3
; CHECK-NEXT:    vldrw.u32 q6, [r6]
; CHECK-NEXT:    add r5, sp, #32
; CHECK-NEXT:    add r6, sp, #64
; CHECK-NEXT:    movs r3, #0
; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
; CHECK-NEXT:  .LBB18_2: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
; CHECK-NEXT:    vdup.32 q7, r3
; CHECK-NEXT:    adds r3, #16
; CHECK-NEXT:    vorr q0, q7, q0
; CHECK-NEXT:    vcmp.u32 cs, q1, q0
; CHECK-NEXT:    vpsel q0, q3, q2
; CHECK-NEXT:    vstrh.32 q0, [r4, #8]
; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
; CHECK-NEXT:    vorr q0, q7, q0
; CHECK-NEXT:    vcmp.u32 cs, q1, q0
; CHECK-NEXT:    vpsel q0, q3, q2
; CHECK-NEXT:    vstrh.32 q0, [r4]
; CHECK-NEXT:    vorr q0, q7, q5
; CHECK-NEXT:    vcmp.u32 cs, q1, q0
; CHECK-NEXT:    vpsel q0, q3, q2
; CHECK-NEXT:    vstrh.32 q0, [r5, #8]
; CHECK-NEXT:    vorr q0, q7, q6
; CHECK-NEXT:    vcmp.u32 cs, q1, q0
; CHECK-NEXT:    vpsel q0, q3, q2
; CHECK-NEXT:    vstrh.32 q0, [r5]
; CHECK-NEXT:    vldrw.u32 q0, [r4]
; CHECK-NEXT:    vcmp.i16 ne, q0, zr
; CHECK-NEXT:    vpsel q0, q3, q2
; CHECK-NEXT:    vstrb.16 q0, [r6, #8]
; CHECK-NEXT:    vldrw.u32 q0, [r5]
; CHECK-NEXT:    vcmp.i16 ne, q0, zr
; CHECK-NEXT:    vpsel q0, q3, q2
; CHECK-NEXT:    vstrb.16 q0, [r6]
; CHECK-NEXT:    vldrw.u32 q0, [r6]
; CHECK-NEXT:    vptt.i8 ne, q0, zr
; CHECK-NEXT:    vldrbt.u8 q0, [r0], #16
; CHECK-NEXT:    vldrbt.u8 q7, [r1], #16
; CHECK-NEXT:    vmullt.s8 q4, q7, q0
; CHECK-NEXT:    vmullb.s8 q0, q7, q0
; CHECK-NEXT:    vqshrnb.s16 q0, q0, #7
; CHECK-NEXT:    vqshrnt.s16 q0, q4, #7
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrbt.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB18_2
; CHECK-NEXT:  .LBB18_3: @ %for.cond.cleanup
; CHECK-NEXT:    add sp, #80
; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    pop {r4, r5, r6, pc}
; CHECK-NEXT:    .p2align 4
; CHECK-NEXT:  @ %bb.4:
; CHECK-NEXT:  .LCPI18_0:
; CHECK-NEXT:    .long 12 @ 0xc
; CHECK-NEXT:    .long 13 @ 0xd
; CHECK-NEXT:    .long 14 @ 0xe
; CHECK-NEXT:    .long 15 @ 0xf
; CHECK-NEXT:  .LCPI18_1:
; CHECK-NEXT:    .long 8 @ 0x8
; CHECK-NEXT:    .long 9 @ 0x9
; CHECK-NEXT:    .long 10 @ 0xa
; CHECK-NEXT:    .long 11 @ 0xb
; CHECK-NEXT:  .LCPI18_2:
; CHECK-NEXT:    .long 4 @ 0x4
; CHECK-NEXT:    .long 5 @ 0x5
; CHECK-NEXT:    .long 6 @ 0x6
; CHECK-NEXT:    .long 7 @ 0x7
; CHECK-NEXT:  .LCPI18_3:
; CHECK-NEXT:    .long 0 @ 0x0
; CHECK-NEXT:    .long 1 @ 0x1
; CHECK-NEXT:    .long 2 @ 0x2
; CHECK-NEXT:    .long 3 @ 0x3
entry:
  %cmp10 = icmp eq i32 %N, 0
  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph

vector.ph:                                        ; preds = %entry
  %n.rnd.up = add i32 %N, 15
  %n.vec = and i32 %n.rnd.up, -16
  %trip.count.minus.1 = add i32 %N, -1
  %broadcast.splatinsert22 = insertelement <16 x i32> undef, i32 %trip.count.minus.1, i32 0
  %broadcast.splat23 = shufflevector <16 x i32> %broadcast.splatinsert22, <16 x i32> undef, <16 x i32> zeroinitializer
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0
  %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer
  %induction = or <16 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  %next.gep = getelementptr i8, ptr %pSrcA, i32 %index
  %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index
  %next.gep21 = getelementptr i8, ptr %pDst, i32 %index
  %0 = icmp ule <16 x i32> %induction, %broadcast.splat23
  %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %next.gep, i32 1, <16 x i1> %0, <16 x i8> undef)
  %1 = sext <16 x i8> %wide.masked.load to <16 x i16>
  %wide.masked.load24 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %next.gep20, i32 1, <16 x i1> %0, <16 x i8> undef)
  %2 = sext <16 x i8> %wide.masked.load24 to <16 x i16>
  %3 = mul nsw <16 x i16> %2, %1
  %4 = ashr <16 x i16> %3, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
  %5 = icmp sgt <16 x i16> %4, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %6 = select <16 x i1> %5, <16 x i16> %4, <16 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %7 = icmp slt <16 x i16> %6, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %8 = select <16 x i1> %7, <16 x i16> %6, <16 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %9 = trunc <16 x i16> %8 to <16 x i8>
  call void @llvm.masked.store.v16i8.p0(<16 x i8> %9, ptr %next.gep21, i32 1, <16 x i1> %0)
  %index.next = add i32 %index, 16
  %10 = icmp eq i32 %index.next, %n.vec
  br i1 %10, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body, %entry
  ret void
}

define arm_aapcs_vfpcc void @ssatmul_16ti_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: ssatmul_16ti_q7:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, lr}
; CHECK-NEXT:    push {r4, r5, r6, lr}
; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    .pad #80
; CHECK-NEXT:    sub sp, #80
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq .LBB19_3
; CHECK-NEXT:  @ %bb.1: @ %vector.ph
; CHECK-NEXT:    add.w r6, r3, #15
; CHECK-NEXT:    movs r5, #1
; CHECK-NEXT:    bic r6, r6, #15
; CHECK-NEXT:    add r4, sp, #48
; CHECK-NEXT:    subs r6, #16
; CHECK-NEXT:    vmov.i8 q2, #0x0
; CHECK-NEXT:    vmov.i8 q3, #0xff
; CHECK-NEXT:    add.w lr, r5, r6, lsr #4
; CHECK-NEXT:    adr r5, .LCPI19_0
; CHECK-NEXT:    subs r6, r3, #1
; CHECK-NEXT:    vldrw.u32 q0, [r5]
; CHECK-NEXT:    vdup.32 q1, r6
; CHECK-NEXT:    adr r6, .LCPI19_1
; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
; CHECK-NEXT:    vldrw.u32 q0, [r6]
; CHECK-NEXT:    adr r6, .LCPI19_2
; CHECK-NEXT:    vldrw.u32 q5, [r6]
; CHECK-NEXT:    adr r6, .LCPI19_3
; CHECK-NEXT:    vldrw.u32 q6, [r6]
; CHECK-NEXT:    add r5, sp, #32
; CHECK-NEXT:    add r6, sp, #64
; CHECK-NEXT:    movs r3, #0
; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
; CHECK-NEXT:  .LBB19_2: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
; CHECK-NEXT:    vdup.32 q7, r3
; CHECK-NEXT:    adds r3, #16
; CHECK-NEXT:    vorr q0, q7, q0
; CHECK-NEXT:    vcmp.u32 cs, q1, q0
; CHECK-NEXT:    vpsel q0, q3, q2
; CHECK-NEXT:    vstrh.32 q0, [r4, #8]
; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
; CHECK-NEXT:    vorr q0, q7, q0
; CHECK-NEXT:    vcmp.u32 cs, q1, q0
; CHECK-NEXT:    vpsel q0, q3, q2
; CHECK-NEXT:    vstrh.32 q0, [r4]
; CHECK-NEXT:    vorr q0, q7, q5
; CHECK-NEXT:    vcmp.u32 cs, q1, q0
; CHECK-NEXT:    vpsel q0, q3, q2
; CHECK-NEXT:    vstrh.32 q0, [r5, #8]
; CHECK-NEXT:    vorr q0, q7, q6
; CHECK-NEXT:    vcmp.u32 cs, q1, q0
; CHECK-NEXT:    vpsel q0, q3, q2
; CHECK-NEXT:    vstrh.32 q0, [r5]
; CHECK-NEXT:    vldrw.u32 q0, [r4]
; CHECK-NEXT:    vcmp.i16 ne, q0, zr
; CHECK-NEXT:    vpsel q0, q3, q2
; CHECK-NEXT:    vstrb.16 q0, [r6, #8]
; CHECK-NEXT:    vldrw.u32 q0, [r5]
; CHECK-NEXT:    vcmp.i16 ne, q0, zr
; CHECK-NEXT:    vpsel q0, q3, q2
; CHECK-NEXT:    vstrb.16 q0, [r6]
; CHECK-NEXT:    vldrw.u32 q0, [r6]
; CHECK-NEXT:    vptt.i8 ne, q0, zr
; CHECK-NEXT:    vldrbt.u8 q0, [r0], #16
; CHECK-NEXT:    vldrbt.u8 q7, [r1], #16
; CHECK-NEXT:    vmullt.s8 q4, q7, q0
; CHECK-NEXT:    vmullb.s8 q0, q7, q0
; CHECK-NEXT:    vqshrnb.s16 q0, q0, #7
; CHECK-NEXT:    vqshrnt.s16 q0, q4, #7
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrbt.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB19_2
; CHECK-NEXT:  .LBB19_3: @ %for.cond.cleanup
; CHECK-NEXT:    add sp, #80
; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT:    pop {r4, r5, r6, pc}
; CHECK-NEXT:    .p2align 4
; CHECK-NEXT:  @ %bb.4:
; CHECK-NEXT:  .LCPI19_0:
; CHECK-NEXT:    .long 12 @ 0xc
; CHECK-NEXT:    .long 13 @ 0xd
; CHECK-NEXT:    .long 14 @ 0xe
; CHECK-NEXT:    .long 15 @ 0xf
; CHECK-NEXT:  .LCPI19_1:
; CHECK-NEXT:    .long 8 @ 0x8
; CHECK-NEXT:    .long 9 @ 0x9
; CHECK-NEXT:    .long 10 @ 0xa
; CHECK-NEXT:    .long 11 @ 0xb
; CHECK-NEXT:  .LCPI19_2:
; CHECK-NEXT:    .long 4 @ 0x4
; CHECK-NEXT:    .long 5 @ 0x5
; CHECK-NEXT:    .long 6 @ 0x6
; CHECK-NEXT:    .long 7 @ 0x7
; CHECK-NEXT:  .LCPI19_3:
; CHECK-NEXT:    .long 0 @ 0x0
; CHECK-NEXT:    .long 1 @ 0x1
; CHECK-NEXT:    .long 2 @ 0x2
; CHECK-NEXT:    .long 3 @ 0x3
entry:
  %cmp10 = icmp eq i32 %N, 0
  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph

vector.ph:                                        ; preds = %entry
  %n.rnd.up = add i32 %N, 15
  %n.vec = and i32 %n.rnd.up, -16
  %trip.count.minus.1 = add i32 %N, -1
  %broadcast.splatinsert22 = insertelement <16 x i32> undef, i32 %trip.count.minus.1, i32 0
  %broadcast.splat23 = shufflevector <16 x i32> %broadcast.splatinsert22, <16 x i32> undef, <16 x i32> zeroinitializer
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0
  %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer
  %induction = or <16 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  %next.gep = getelementptr i8, ptr %pSrcA, i32 %index
  %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index
  %next.gep21 = getelementptr i8, ptr %pDst, i32 %index
  %0 = icmp ule <16 x i32> %induction, %broadcast.splat23
  %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %next.gep, i32 1, <16 x i1> %0, <16 x i8> undef)
  %1 = shufflevector <16 x i8> %wide.masked.load, <16 x i8> %wide.masked.load, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
  %2 = shufflevector <16 x i8> %wide.masked.load, <16 x i8> %wide.masked.load, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
  %3 = sext <8 x i8> %1 to <8 x i16>
  %4 = sext <8 x i8> %2 to <8 x i16>
  %wide.masked.load24 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %next.gep20, i32 1, <16 x i1> %0, <16 x i8> undef)
  %5 = shufflevector <16 x i8> %wide.masked.load24, <16 x i8> %wide.masked.load24, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
  %6 = shufflevector <16 x i8> %wide.masked.load24, <16 x i8> %wide.masked.load24, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
  %7 = sext <8 x i8> %5 to <8 x i16>
  %8 = sext <8 x i8> %6 to <8 x i16>
  %9 = mul <8 x i16> %7, %3
  %10 = mul <8 x i16> %8, %4
  %11 = ashr <8 x i16> %9, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
  %12 = ashr <8 x i16> %10, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
  %13 = icmp sgt <8 x i16> %11, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %14 = icmp sgt <8 x i16> %12, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %15 = select <8 x i1> %13, <8 x i16> %11, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %16 = select <8 x i1> %14, <8 x i16> %12, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
  %17 = icmp slt <8 x i16> %15, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %18 = icmp slt <8 x i16> %16, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %19 = select <8 x i1> %17, <8 x i16> %15, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %20 = select <8 x i1> %18, <8 x i16> %16, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
  %21 = shufflevector <8 x i16> %19, <8 x i16> %20, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
  %22 = trunc <16 x i16> %21 to <16 x i8>
  call void @llvm.masked.store.v16i8.p0(<16 x i8> %22, ptr %next.gep21, i32 1, <16 x i1> %0)
  %index.next = add i32 %index, 16
  %23 = icmp eq i32 %index.next, %n.vec
  br i1 %23, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body, %entry
  ret void
}

define arm_aapcs_vfpcc void @usatmul_8_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: usatmul_8_q7:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, lr}
; CHECK-NEXT:    push {r4, r5, r6, lr}
; CHECK-NEXT:    cbz r3, .LBB20_8
; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
; CHECK-NEXT:    cmp r3, #7
; CHECK-NEXT:    bhi .LBB20_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    movs r5, #0
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r6, r1
; CHECK-NEXT:    mov r4, r2
; CHECK-NEXT:    b .LBB20_6
; CHECK-NEXT:  .LBB20_3: @ %vector.ph
; CHECK-NEXT:    bic r5, r3, #7
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    sub.w r6, r5, #8
; CHECK-NEXT:    add.w r12, r0, r5
; CHECK-NEXT:    add.w lr, r4, r6, lsr #3
; CHECK-NEXT:    adds r4, r2, r5
; CHECK-NEXT:    adds r6, r1, r5
; CHECK-NEXT:  .LBB20_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u16 q0, [r0], #8
; CHECK-NEXT:    vldrb.u16 q1, [r1], #8
; CHECK-NEXT:    vmul.i16 q0, q1, q0
; CHECK-NEXT:    vqshrnb.u16 q0, q0, #7
; CHECK-NEXT:    vstrb.16 q0, [r2], #8
; CHECK-NEXT:    le lr, .LBB20_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    cmp r5, r3
; CHECK-NEXT:    it eq
; CHECK-NEXT:    popeq {r4, r5, r6, pc}
; CHECK-NEXT:  .LBB20_6: @ %for.body.preheader23
; CHECK-NEXT:    sub.w lr, r3, r5
; CHECK-NEXT:  .LBB20_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldrb r0, [r12], #1
; CHECK-NEXT:    ldrb r1, [r6], #1
; CHECK-NEXT:    muls r0, r1, r0
; CHECK-NEXT:    lsrs r1, r0, #7
; CHECK-NEXT:    cmp r1, #255
; CHECK-NEXT:    mov.w r1, #255
; CHECK-NEXT:    it lo
; CHECK-NEXT:    lsrlo r1, r0, #7
; CHECK-NEXT:    strb r1, [r4], #1
; CHECK-NEXT:    le lr, .LBB20_7
; CHECK-NEXT:  .LBB20_8: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r4, r5, r6, pc}
entry:
  %cmp10 = icmp eq i32 %N, 0
  br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry
  %min.iters.check = icmp ult i32 %N, 8
  br i1 %min.iters.check, label %for.body.preheader23, label %vector.ph

for.body.preheader23:                             ; preds = %middle.block, %for.body.preheader
  %i.014.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
  %pSrcA.addr.013.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ]
  %pSrcB.addr.012.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end17, %middle.block ]
  %pDst.addr.011.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end19, %middle.block ]
  br label %for.body

vector.ph:                                        ; preds = %for.body.preheader
  %n.vec = and i32 %N, -8
  %ind.end = getelementptr i8, ptr %pSrcA, i32 %n.vec
  %ind.end17 = getelementptr i8, ptr %pSrcB, i32 %n.vec
  %ind.end19 = getelementptr i8, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i8, ptr %pSrcA, i32 %index
  %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index
  %next.gep21 = getelementptr i8, ptr %pDst, i32 %index
  %wide.load = load <8 x i8>, ptr %next.gep, align 1
  %0 = zext <8 x i8> %wide.load to <8 x i16>
  %wide.load22 = load <8 x i8>, ptr %next.gep20, align 1
  %1 = zext <8 x i8> %wide.load22 to <8 x i16>
  %2 = mul nuw <8 x i16> %1, %0
  %3 = lshr <8 x i16> %2, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
  %4 = icmp ult <8 x i16> %3, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
  %5 = select <8 x i1> %4, <8 x i16> %3, <8 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
  %6 = trunc <8 x i16> %5 to <8 x i8>
  store <8 x i8> %6, ptr %next.gep21, align 1
  %index.next = add i32 %index, 8
  %7 = icmp eq i32 %index.next, %n.vec
  br i1 %7, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader23

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body.preheader23, %for.body
  %i.014 = phi i32 [ %inc, %for.body ], [ %i.014.ph, %for.body.preheader23 ]
  %pSrcA.addr.013 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.013.ph, %for.body.preheader23 ]
  %pSrcB.addr.012 = phi ptr [ %incdec.ptr2, %for.body ], [ %pSrcB.addr.012.ph, %for.body.preheader23 ]
  %pDst.addr.011 = phi ptr [ %incdec.ptr6, %for.body ], [ %pDst.addr.011.ph, %for.body.preheader23 ]
  %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.013, i32 1
  %8 = load i8, ptr %pSrcA.addr.013, align 1
  %conv1 = zext i8 %8 to i16
  %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.012, i32 1
  %9 = load i8, ptr %pSrcB.addr.012, align 1
  %conv3 = zext i8 %9 to i16
  %mul = mul nuw i16 %conv3, %conv1
  %10 = lshr i16 %mul, 7
  %11 = icmp ult i16 %10, 255
  %retval.0.i = select i1 %11, i16 %10, i16 255
  %conv5 = trunc i16 %retval.0.i to i8
  %incdec.ptr6 = getelementptr inbounds i8, ptr %pDst.addr.011, i32 1
  store i8 %conv5, ptr %pDst.addr.011, align 1
  %inc = add nuw i32 %i.014, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}

define arm_aapcs_vfpcc void @usatmul_16_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
; CHECK-LABEL: usatmul_16_q7:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r4, r5, r6, lr}
; CHECK-NEXT:    push {r4, r5, r6, lr}
; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    beq .LBB21_8
; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
; CHECK-NEXT:    cmp r3, #15
; CHECK-NEXT:    bhi .LBB21_3
; CHECK-NEXT:  @ %bb.2:
; CHECK-NEXT:    movs r5, #0
; CHECK-NEXT:    mov r12, r0
; CHECK-NEXT:    mov r6, r1
; CHECK-NEXT:    mov r4, r2
; CHECK-NEXT:    b .LBB21_6
; CHECK-NEXT:  .LBB21_3: @ %vector.ph
; CHECK-NEXT:    bic r5, r3, #15
; CHECK-NEXT:    movs r4, #1
; CHECK-NEXT:    sub.w r6, r5, #16
; CHECK-NEXT:    add.w r12, r0, r5
; CHECK-NEXT:    add.w lr, r4, r6, lsr #4
; CHECK-NEXT:    adds r4, r2, r5
; CHECK-NEXT:    adds r6, r1, r5
; CHECK-NEXT:  .LBB21_4: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
; CHECK-NEXT:    vmullt.u8 q2, q1, q0
; CHECK-NEXT:    vmullb.u8 q0, q1, q0
; CHECK-NEXT:    vqshrnb.u16 q0, q0, #7
; CHECK-NEXT:    vqshrnt.u16 q0, q2, #7
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB21_4
; CHECK-NEXT:  @ %bb.5: @ %middle.block
; CHECK-NEXT:    cmp r5, r3
; CHECK-NEXT:    it eq
; CHECK-NEXT:    popeq {r4, r5, r6, pc}
; CHECK-NEXT:  .LBB21_6: @ %for.body.preheader23
; CHECK-NEXT:    sub.w lr, r3, r5
; CHECK-NEXT:  .LBB21_7: @ %for.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldrb r0, [r12], #1
; CHECK-NEXT:    ldrb r1, [r6], #1
; CHECK-NEXT:    muls r0, r1, r0
; CHECK-NEXT:    lsrs r1, r0, #7
; CHECK-NEXT:    cmp r1, #255
; CHECK-NEXT:    mov.w r1, #255
; CHECK-NEXT:    it lo
; CHECK-NEXT:    lsrlo r1, r0, #7
; CHECK-NEXT:    strb r1, [r4], #1
; CHECK-NEXT:    le lr, .LBB21_7
; CHECK-NEXT:  .LBB21_8: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r4, r5, r6, pc}
entry:
  %cmp10 = icmp eq i32 %N, 0
  br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader

for.body.preheader:                               ; preds = %entry
  %min.iters.check = icmp ult i32 %N, 16
  br i1 %min.iters.check, label %for.body.preheader23, label %vector.ph

for.body.preheader23:                             ; preds = %middle.block, %for.body.preheader
  %i.014.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
  %pSrcA.addr.013.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ]
  %pSrcB.addr.012.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end17, %middle.block ]
  %pDst.addr.011.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end19, %middle.block ]
  br label %for.body

vector.ph:                                        ; preds = %for.body.preheader
  %n.vec = and i32 %N, -16
  %ind.end = getelementptr i8, ptr %pSrcA, i32 %n.vec
  %ind.end17 = getelementptr i8, ptr %pSrcB, i32 %n.vec
  %ind.end19 = getelementptr i8, ptr %pDst, i32 %n.vec
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %vector.ph
  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
  %next.gep = getelementptr i8, ptr %pSrcA, i32 %index
  %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index
  %next.gep21 = getelementptr i8, ptr %pDst, i32 %index
  %wide.load = load <16 x i8>, ptr %next.gep, align 1
  %0 = zext <16 x i8> %wide.load to <16 x i16>
  %wide.load22 = load <16 x i8>, ptr %next.gep20, align 1
  %1 = zext <16 x i8> %wide.load22 to <16 x i16>
  %2 = mul nuw <16 x i16> %1, %0
  %3 = lshr <16 x i16> %2, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
  %4 = icmp ult <16 x i16> %3, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
  %5 = select <16 x i1> %4, <16 x i16> %3, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
  %6 = trunc <16 x i16> %5 to <16 x i8>
  store <16 x i8> %6, ptr %next.gep21, align 1
  %index.next = add i32 %index, 16
  %7 = icmp eq i32 %index.next, %n.vec
  br i1 %7, label %middle.block, label %vector.body

middle.block:                                     ; preds = %vector.body
  %cmp.n = icmp eq i32 %n.vec, %N
  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader23

for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
  ret void

for.body:                                         ; preds = %for.body.preheader23, %for.body
  %i.014 = phi i32 [ %inc, %for.body ], [ %i.014.ph, %for.body.preheader23 ]
  %pSrcA.addr.013 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.013.ph, %for.body.preheader23 ]
  %pSrcB.addr.012 = phi ptr [ %incdec.ptr2, %for.body ], [ %pSrcB.addr.012.ph, %for.body.preheader23 ]
  %pDst.addr.011 = phi ptr [ %incdec.ptr6, %for.body ], [ %pDst.addr.011.ph, %for.body.preheader23 ]
  %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.013, i32 1
  %8 = load i8, ptr %pSrcA.addr.013, align 1
  %conv1 = zext i8 %8 to i16
  %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.012, i32 1
  %9 = load i8, ptr %pSrcB.addr.012, align 1
  %conv3 = zext i8 %9 to i16
  %mul = mul nuw i16 %conv3, %conv1
  %10 = lshr i16 %mul, 7
  %11 = icmp ult i16 %10, 255
  %retval.0.i = select i1 %11, i16 %10, i16 255
  %conv5 = trunc i16 %retval.0.i to i8
  %incdec.ptr6 = getelementptr inbounds i8, ptr %pDst.addr.011, i32 1
  store i8 %conv5, ptr %pDst.addr.011, align 1
  %inc = add nuw i32 %i.014, 1
  %exitcond = icmp eq i32 %inc, %N
  br i1 %exitcond, label %for.cond.cleanup, label %for.body
}

declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32, <4 x i1>, <4 x i32>)
declare <8 x i16> @llvm.masked.load.v8i16.p0(ptr, i32, <8 x i1>, <8 x i16>)
declare <4 x i16> @llvm.masked.load.v4i16.p0(ptr, i32, <4 x i1>, <4 x i16>)
declare <8 x i8> @llvm.masked.load.v8i8.p0(ptr, i32, <8 x i1>, <8 x i8>)
declare <16 x i8> @llvm.masked.load.v16i8.p0(ptr, i32, <16 x i1>, <16 x i8>)
declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32, <4 x i1>)
declare void @llvm.masked.store.v8i16.p0(<8 x i16>, ptr, i32, <8 x i1>)
declare void @llvm.masked.store.v4i16.p0(<4 x i16>, ptr, i32, <4 x i1>)
declare void @llvm.masked.store.v8i8.p0(<8 x i8>, ptr, i32, <8 x i1>)
declare void @llvm.masked.store.v16i8.p0(<16 x i8>, ptr, i32, <16 x i1>)