; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK

define arm_aapcs_vfpcc <4 x i32> @vhadds_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
; CHECK-LABEL: vhadds_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.s32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i32> %s0 to <4 x i64>
  %s1s = sext <4 x i32> %s1 to <4 x i64>
  %m = add <4 x i64> %s0s, %s1s
  %s = lshr <4 x i64> %m, <i64 1, i64 1, i64 1, i64 1>
  %s2 = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %s2
}

define arm_aapcs_vfpcc <4 x i32> @vhaddu_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
; CHECK-LABEL: vhaddu_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.u32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i32> %s0 to <4 x i64>
  %s1s = zext <4 x i32> %s1 to <4 x i64>
  %m = add <4 x i64> %s0s, %s1s
  %s = lshr <4 x i64> %m, <i64 1, i64 1, i64 1, i64 1>
  %s2 = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %s2
}

define arm_aapcs_vfpcc <4 x i16> @vhadds_v4i16(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-LABEL: vhadds_v4i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.s16 q1, q1
; CHECK-NEXT:    vmovlb.s16 q0, q0
; CHECK-NEXT:    vadd.i32 q0, q0, q1
; CHECK-NEXT:    vshr.u32 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i16> %s0 to <4 x i32>
  %s1s = sext <4 x i16> %s1 to <4 x i32>
  %m = add <4 x i32> %s0s, %s1s
  %s = lshr <4 x i32> %m, <i32 1, i32 1, i32 1, i32 1>
  %s2 = trunc <4 x i32> %s to <4 x i16>
  ret <4 x i16> %s2
}

define arm_aapcs_vfpcc <4 x i16> @vhaddu_v4i16(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-LABEL: vhaddu_v4i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.u16 q1, q1
; CHECK-NEXT:    vmovlb.u16 q0, q0
; CHECK-NEXT:    vadd.i32 q0, q0, q1
; CHECK-NEXT:    vshr.u32 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i16> %s0 to <4 x i32>
  %s1s = zext <4 x i16> %s1 to <4 x i32>
  %m = add <4 x i32> %s0s, %s1s
  %s = lshr <4 x i32> %m, <i32 1, i32 1, i32 1, i32 1>
  %s2 = trunc <4 x i32> %s to <4 x i16>
  ret <4 x i16> %s2
}

define arm_aapcs_vfpcc <8 x i16> @vhadds_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
; CHECK-LABEL: vhadds_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.s16 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <8 x i16> %s0 to <8 x i32>
  %s1s = sext <8 x i16> %s1 to <8 x i32>
  %m = add <8 x i32> %s0s, %s1s
  %s = lshr <8 x i32> %m, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %s2 = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %s2
}

define arm_aapcs_vfpcc <8 x i16> @vhaddu_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
; CHECK-LABEL: vhaddu_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.u16 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <8 x i16> %s0 to <8 x i32>
  %s1s = zext <8 x i16> %s1 to <8 x i32>
  %m = add <8 x i32> %s0s, %s1s
  %s = lshr <8 x i32> %m, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %s2 = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %s2
}

define arm_aapcs_vfpcc <4 x i8> @vhadds_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
; CHECK-LABEL: vhadds_v4i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.s8 q1, q1
; CHECK-NEXT:    vmovlb.s8 q0, q0
; CHECK-NEXT:    vmovlb.s16 q1, q1
; CHECK-NEXT:    vmovlb.s16 q0, q0
; CHECK-NEXT:    vadd.i32 q0, q0, q1
; CHECK-NEXT:    vmovlb.u16 q0, q0
; CHECK-NEXT:    vshr.u32 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i8> %s0 to <4 x i16>
  %s1s = sext <4 x i8> %s1 to <4 x i16>
  %m = add <4 x i16> %s0s, %s1s
  %s = lshr <4 x i16> %m, <i16 1, i16 1, i16 1, i16 1>
  %s2 = trunc <4 x i16> %s to <4 x i8>
  ret <4 x i8> %s2
}

define arm_aapcs_vfpcc <4 x i8> @vhaddu_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
; CHECK-LABEL: vhaddu_v4i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmov.i32 q2, #0xff
; CHECK-NEXT:    vand q1, q1, q2
; CHECK-NEXT:    vand q0, q0, q2
; CHECK-NEXT:    vadd.i32 q0, q0, q1
; CHECK-NEXT:    vshr.u32 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i8> %s0 to <4 x i16>
  %s1s = zext <4 x i8> %s1 to <4 x i16>
  %m = add <4 x i16> %s0s, %s1s
  %s = lshr <4 x i16> %m, <i16 1, i16 1, i16 1, i16 1>
  %s2 = trunc <4 x i16> %s to <4 x i8>
  ret <4 x i8> %s2
}

define arm_aapcs_vfpcc <8 x i8> @vhadds_v8i8(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-LABEL: vhadds_v8i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.s8 q1, q1
; CHECK-NEXT:    vmovlb.s8 q0, q0
; CHECK-NEXT:    vadd.i16 q0, q0, q1
; CHECK-NEXT:    vshr.u16 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <8 x i8> %s0 to <8 x i16>
  %s1s = sext <8 x i8> %s1 to <8 x i16>
  %m = add <8 x i16> %s0s, %s1s
  %s = lshr <8 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s2 = trunc <8 x i16> %s to <8 x i8>
  ret <8 x i8> %s2
}

define arm_aapcs_vfpcc <8 x i8> @vhaddu_v8i8(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-LABEL: vhaddu_v8i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.u8 q1, q1
; CHECK-NEXT:    vmovlb.u8 q0, q0
; CHECK-NEXT:    vadd.i16 q0, q0, q1
; CHECK-NEXT:    vshr.u16 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <8 x i8> %s0 to <8 x i16>
  %s1s = zext <8 x i8> %s1 to <8 x i16>
  %m = add <8 x i16> %s0s, %s1s
  %s = lshr <8 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s2 = trunc <8 x i16> %s to <8 x i8>
  ret <8 x i8> %s2
}

define arm_aapcs_vfpcc <16 x i8> @vhadds_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vhadds_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.s8 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <16 x i8> %s0 to <16 x i16>
  %s1s = sext <16 x i8> %s1 to <16 x i16>
  %m = add <16 x i16> %s0s, %s1s
  %s = lshr <16 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s2 = trunc <16 x i16> %s to <16 x i8>
  ret <16 x i8> %s2
}

define arm_aapcs_vfpcc <16 x i8> @vhaddu_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vhaddu_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.u8 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <16 x i8> %s0 to <16 x i16>
  %s1s = zext <16 x i8> %s1 to <16 x i16>
  %m = add <16 x i16> %s0s, %s1s
  %s = lshr <16 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s2 = trunc <16 x i16> %s to <16 x i8>
  ret <16 x i8> %s2
}

define arm_aapcs_vfpcc <4 x i32> @vrhadds_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
; CHECK-LABEL: vrhadds_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.s32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i32> %s0 to <4 x i64>
  %s1s = sext <4 x i32> %s1 to <4 x i64>
  %add = add <4 x i64> %s0s, %s1s
  %add2 = add <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>
  %s = lshr <4 x i64> %add2, <i64 1, i64 1, i64 1, i64 1>
  %result = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %result
}

define arm_aapcs_vfpcc <4 x i32> @vrhaddu_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
; CHECK-LABEL: vrhaddu_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.u32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i32> %s0 to <4 x i64>
  %s1s = zext <4 x i32> %s1 to <4 x i64>
  %add = add <4 x i64> %s0s, %s1s
  %add2 = add <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>
  %s = lshr <4 x i64> %add2, <i64 1, i64 1, i64 1, i64 1>
  %result = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %result
}

define arm_aapcs_vfpcc <4 x i16> @vrhadds_v4i16(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-LABEL: vrhadds_v4i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.s16 q1, q1
; CHECK-NEXT:    vmovlb.s16 q0, q0
; CHECK-NEXT:    vadd.i32 q0, q0, q1
; CHECK-NEXT:    movs r0, #1
; CHECK-NEXT:    vadd.i32 q0, q0, r0
; CHECK-NEXT:    vshr.u32 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i16> %s0 to <4 x i32>
  %s1s = sext <4 x i16> %s1 to <4 x i32>
  %add = add <4 x i32> %s0s, %s1s
  %add2 = add <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>
  %s = lshr <4 x i32> %add2, <i32 1, i32 1, i32 1, i32 1>
  %result = trunc <4 x i32> %s to <4 x i16>
  ret <4 x i16> %result
}

define arm_aapcs_vfpcc <4 x i16> @vrhaddu_v4i16(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-LABEL: vrhaddu_v4i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.u16 q1, q1
; CHECK-NEXT:    vmovlb.u16 q0, q0
; CHECK-NEXT:    vadd.i32 q0, q0, q1
; CHECK-NEXT:    movs r0, #1
; CHECK-NEXT:    vadd.i32 q0, q0, r0
; CHECK-NEXT:    vshr.u32 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i16> %s0 to <4 x i32>
  %s1s = zext <4 x i16> %s1 to <4 x i32>
  %add = add <4 x i32> %s0s, %s1s
  %add2 = add <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>
  %s = lshr <4 x i32> %add2, <i32 1, i32 1, i32 1, i32 1>
  %result = trunc <4 x i32> %s to <4 x i16>
  ret <4 x i16> %result
}

define arm_aapcs_vfpcc <8 x i16> @vrhadds_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
; CHECK-LABEL: vrhadds_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.s16 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <8 x i16> %s0 to <8 x i32>
  %s1s = sext <8 x i16> %s1 to <8 x i32>
  %add = add <8 x i32> %s0s, %s1s
  %add2 = add <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %s = lshr <8 x i32> %add2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %result = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %result
}

define arm_aapcs_vfpcc <8 x i16> @vrhaddu_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
; CHECK-LABEL: vrhaddu_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.u16 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <8 x i16> %s0 to <8 x i32>
  %s1s = zext <8 x i16> %s1 to <8 x i32>
  %add = add <8 x i32> %s0s, %s1s
  %add2 = add <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %s = lshr <8 x i32> %add2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %result = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %result
}

define arm_aapcs_vfpcc <4 x i8> @vrhadds_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
; CHECK-LABEL: vrhadds_v4i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.s8 q1, q1
; CHECK-NEXT:    vmovlb.s8 q0, q0
; CHECK-NEXT:    vmovlb.s16 q1, q1
; CHECK-NEXT:    vmovlb.s16 q0, q0
; CHECK-NEXT:    vadd.i32 q0, q0, q1
; CHECK-NEXT:    movs r0, #1
; CHECK-NEXT:    vadd.i32 q0, q0, r0
; CHECK-NEXT:    vmovlb.u16 q0, q0
; CHECK-NEXT:    vshr.u32 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i8> %s0 to <4 x i16>
  %s1s = sext <4 x i8> %s1 to <4 x i16>
  %add = add <4 x i16> %s0s, %s1s
  %add2 = add <4 x i16> %add, <i16 1, i16 1, i16 1, i16 1>
  %s = lshr <4 x i16> %add2, <i16 1, i16 1, i16 1, i16 1>
  %result = trunc <4 x i16> %s to <4 x i8>
  ret <4 x i8> %result
}

define arm_aapcs_vfpcc <4 x i8> @vrhaddu_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
; CHECK-LABEL: vrhaddu_v4i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmov.i32 q2, #0xff
; CHECK-NEXT:    movs r0, #1
; CHECK-NEXT:    vand q1, q1, q2
; CHECK-NEXT:    vand q0, q0, q2
; CHECK-NEXT:    vadd.i32 q0, q0, q1
; CHECK-NEXT:    vadd.i32 q0, q0, r0
; CHECK-NEXT:    vshr.u32 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i8> %s0 to <4 x i16>
  %s1s = zext <4 x i8> %s1 to <4 x i16>
  %add = add <4 x i16> %s0s, %s1s
  %add2 = add <4 x i16> %add, <i16 1, i16 1, i16 1, i16 1>
  %s = lshr <4 x i16> %add2, <i16 1, i16 1, i16 1, i16 1>
  %result = trunc <4 x i16> %s to <4 x i8>
  ret <4 x i8> %result
}

define arm_aapcs_vfpcc <8 x i8> @vrhadds_v8i8(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-LABEL: vrhadds_v8i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.s8 q1, q1
; CHECK-NEXT:    vmovlb.s8 q0, q0
; CHECK-NEXT:    vadd.i16 q0, q0, q1
; CHECK-NEXT:    movs r0, #1
; CHECK-NEXT:    vadd.i16 q0, q0, r0
; CHECK-NEXT:    vshr.u16 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <8 x i8> %s0 to <8 x i16>
  %s1s = sext <8 x i8> %s1 to <8 x i16>
  %add = add <8 x i16> %s0s, %s1s
  %add2 = add <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s = lshr <8 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %result = trunc <8 x i16> %s to <8 x i8>
  ret <8 x i8> %result
}

define arm_aapcs_vfpcc <8 x i8> @vrhaddu_v8i8(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-LABEL: vrhaddu_v8i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.u8 q1, q1
; CHECK-NEXT:    vmovlb.u8 q0, q0
; CHECK-NEXT:    vadd.i16 q0, q0, q1
; CHECK-NEXT:    movs r0, #1
; CHECK-NEXT:    vadd.i16 q0, q0, r0
; CHECK-NEXT:    vshr.u16 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <8 x i8> %s0 to <8 x i16>
  %s1s = zext <8 x i8> %s1 to <8 x i16>
  %add = add <8 x i16> %s0s, %s1s
  %add2 = add <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s = lshr <8 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %result = trunc <8 x i16> %s to <8 x i8>
  ret <8 x i8> %result
}

define arm_aapcs_vfpcc <16 x i8> @vrhadds_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vrhadds_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.s8 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <16 x i8> %s0 to <16 x i16>
  %s1s = sext <16 x i8> %s1 to <16 x i16>
  %add = add <16 x i16> %s0s, %s1s
  %add2 = add <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %result = trunc <16 x i16> %s to <16 x i8>
  ret <16 x i8> %result
}

define arm_aapcs_vfpcc <16 x i8> @vrhaddu_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vrhaddu_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.u8 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <16 x i8> %s0 to <16 x i16>
  %s1s = zext <16 x i8> %s1 to <16 x i16>
  %add = add <16 x i16> %s0s, %s1s
  %add2 = add <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %result = trunc <16 x i16> %s to <16 x i8>
  ret <16 x i8> %result
}


define void @vhadd_loop_s8(i8* nocapture readonly %x, i8* nocapture readonly %y, i8* noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vhadd_loop_s8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #64
; CHECK-NEXT:  .LBB24_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
; CHECK-NEXT:    vhadd.s8 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB24_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i8, i8* %x, i32 %index
  %1 = bitcast i8* %0 to <16 x i8>*
  %wide.load = load <16 x i8>, <16 x i8>* %1, align 1
  %2 = sext <16 x i8> %wide.load to <16 x i16>
  %3 = getelementptr inbounds i8, i8* %y, i32 %index
  %4 = bitcast i8* %3 to <16 x i8>*
  %wide.load16 = load <16 x i8>, <16 x i8>* %4, align 1
  %5 = sext <16 x i8> %wide.load16 to <16 x i16>
  %6 = add nsw <16 x i16> %5, %2
  %7 = lshr <16 x i16> %6, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %8 = trunc <16 x i16> %7 to <16 x i8>
  %9 = getelementptr inbounds i8, i8* %z, i32 %index
  %10 = bitcast i8* %9 to <16 x i8>*
  store <16 x i8> %8, <16 x i8>* %10, align 1
  %index.next = add i32 %index, 16
  %11 = icmp eq i32 %index.next, 1024
  br i1 %11, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vhadd_loop_s16(i16* nocapture readonly %x, i16* nocapture readonly %y, i16* noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vhadd_loop_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #128
; CHECK-NEXT:  .LBB25_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
; CHECK-NEXT:    vhadd.s16 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB25_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i16, i16* %x, i32 %index
  %1 = bitcast i16* %0 to <8 x i16>*
  %wide.load = load <8 x i16>, <8 x i16>* %1, align 2
  %2 = sext <8 x i16> %wide.load to <8 x i32>
  %3 = getelementptr inbounds i16, i16* %y, i32 %index
  %4 = bitcast i16* %3 to <8 x i16>*
  %wide.load16 = load <8 x i16>, <8 x i16>* %4, align 2
  %5 = sext <8 x i16> %wide.load16 to <8 x i32>
  %6 = add nsw <8 x i32> %5, %2
  %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %8 = trunc <8 x i32> %7 to <8 x i16>
  %9 = getelementptr inbounds i16, i16* %z, i32 %index
  %10 = bitcast i16* %9 to <8 x i16>*
  store <8 x i16> %8, <8 x i16>* %10, align 2
  %index.next = add i32 %index, 8
  %11 = icmp eq i32 %index.next, 1024
  br i1 %11, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vhadd_loop_s32(i32* nocapture readonly %x, i32* nocapture readonly %y, i32* noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vhadd_loop_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #256
; CHECK-NEXT:  .LBB26_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
; CHECK-NEXT:    vhadd.s32 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB26_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i32, i32* %x, i32 %index
  %1 = bitcast i32* %0 to <4 x i32>*
  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
  %2 = sext <4 x i32> %wide.load to <4 x i64>
  %3 = getelementptr inbounds i32, i32* %y, i32 %index
  %4 = bitcast i32* %3 to <4 x i32>*
  %wide.load16 = load <4 x i32>, <4 x i32>* %4, align 4
  %5 = sext <4 x i32> %wide.load16 to <4 x i64>
  %6 = add nsw <4 x i64> %5, %2
  %7 = lshr <4 x i64> %6, <i64 1, i64 1, i64 1, i64 1>
  %8 = trunc <4 x i64> %7 to <4 x i32>
  %9 = getelementptr inbounds i32, i32* %z, i32 %index
  %10 = bitcast i32* %9 to <4 x i32>*
  store <4 x i32> %8, <4 x i32>* %10, align 4
  %index.next = add i32 %index, 4
  %11 = icmp eq i32 %index.next, 1024
  br i1 %11, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vhadd_loop_u8(i8* nocapture readonly %x, i8* nocapture readonly %y, i8* noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vhadd_loop_u8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #64
; CHECK-NEXT:  .LBB27_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
; CHECK-NEXT:    vhadd.u8 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB27_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i8, i8* %x, i32 %index
  %1 = bitcast i8* %0 to <16 x i8>*
  %wide.load = load <16 x i8>, <16 x i8>* %1, align 1
  %2 = zext <16 x i8> %wide.load to <16 x i16>
  %3 = getelementptr inbounds i8, i8* %y, i32 %index
  %4 = bitcast i8* %3 to <16 x i8>*
  %wide.load16 = load <16 x i8>, <16 x i8>* %4, align 1
  %5 = zext <16 x i8> %wide.load16 to <16 x i16>
  %6 = add nuw nsw <16 x i16> %5, %2
  %7 = lshr <16 x i16> %6, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %8 = trunc <16 x i16> %7 to <16 x i8>
  %9 = getelementptr inbounds i8, i8* %z, i32 %index
  %10 = bitcast i8* %9 to <16 x i8>*
  store <16 x i8> %8, <16 x i8>* %10, align 1
  %index.next = add i32 %index, 16
  %11 = icmp eq i32 %index.next, 1024
  br i1 %11, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vhadd_loop_u16(i16* nocapture readonly %x, i16* nocapture readonly %y, i16* noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vhadd_loop_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #128
; CHECK-NEXT:  .LBB28_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
; CHECK-NEXT:    vhadd.u16 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB28_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i16, i16* %x, i32 %index
  %1 = bitcast i16* %0 to <8 x i16>*
  %wide.load = load <8 x i16>, <8 x i16>* %1, align 2
  %2 = zext <8 x i16> %wide.load to <8 x i32>
  %3 = getelementptr inbounds i16, i16* %y, i32 %index
  %4 = bitcast i16* %3 to <8 x i16>*
  %wide.load16 = load <8 x i16>, <8 x i16>* %4, align 2
  %5 = zext <8 x i16> %wide.load16 to <8 x i32>
  %6 = add nuw nsw <8 x i32> %5, %2
  %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %8 = trunc <8 x i32> %7 to <8 x i16>
  %9 = getelementptr inbounds i16, i16* %z, i32 %index
  %10 = bitcast i16* %9 to <8 x i16>*
  store <8 x i16> %8, <8 x i16>* %10, align 2
  %index.next = add i32 %index, 8
  %11 = icmp eq i32 %index.next, 1024
  br i1 %11, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vhadd_loop_u32(i32* nocapture readonly %x, i32* nocapture readonly %y, i32* noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vhadd_loop_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #256
; CHECK-NEXT:  .LBB29_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
; CHECK-NEXT:    vhadd.u32 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB29_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i32, i32* %x, i32 %index
  %1 = bitcast i32* %0 to <4 x i32>*
  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
  %2 = zext <4 x i32> %wide.load to <4 x i64>
  %3 = getelementptr inbounds i32, i32* %y, i32 %index
  %4 = bitcast i32* %3 to <4 x i32>*
  %wide.load16 = load <4 x i32>, <4 x i32>* %4, align 4
  %5 = zext <4 x i32> %wide.load16 to <4 x i64>
  %6 = add nuw nsw <4 x i64> %5, %2
  %7 = lshr <4 x i64> %6, <i64 1, i64 1, i64 1, i64 1>
  %8 = trunc <4 x i64> %7 to <4 x i32>
  %9 = getelementptr inbounds i32, i32* %z, i32 %index
  %10 = bitcast i32* %9 to <4 x i32>*
  store <4 x i32> %8, <4 x i32>* %10, align 4
  %index.next = add i32 %index, 4
  %11 = icmp eq i32 %index.next, 1024
  br i1 %11, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vrhadd_loop_s8(i8* nocapture readonly %x, i8* nocapture readonly %y, i8* noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vrhadd_loop_s8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #64
; CHECK-NEXT:  .LBB30_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r1], #16
; CHECK-NEXT:    vldrb.u8 q1, [r0], #16
; CHECK-NEXT:    vrhadd.u8 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB30_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i8, i8* %x, i32 %index
  %1 = bitcast i8* %0 to <16 x i8>*
  %wide.load = load <16 x i8>, <16 x i8>* %1, align 1
  %2 = zext <16 x i8> %wide.load to <16 x i16>
  %3 = getelementptr inbounds i8, i8* %y, i32 %index
  %4 = bitcast i8* %3 to <16 x i8>*
  %wide.load16 = load <16 x i8>, <16 x i8>* %4, align 1
  %5 = zext <16 x i8> %wide.load16 to <16 x i16>
  %6 = add nuw nsw <16 x i16> %2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %7 = add nuw nsw <16 x i16> %6, %5
  %8 = lshr <16 x i16> %7, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %9 = trunc <16 x i16> %8 to <16 x i8>
  %10 = getelementptr inbounds i8, i8* %z, i32 %index
  %11 = bitcast i8* %10 to <16 x i8>*
  store <16 x i8> %9, <16 x i8>* %11, align 1
  %index.next = add i32 %index, 16
  %12 = icmp eq i32 %index.next, 1024
  br i1 %12, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vrhadd_loop_s16(i16* nocapture readonly %x, i16* nocapture readonly %y, i16* noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vrhadd_loop_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #128
; CHECK-NEXT:  .LBB31_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
; CHECK-NEXT:    vrhadd.u16 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB31_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i16, i16* %x, i32 %index
  %1 = bitcast i16* %0 to <8 x i16>*
  %wide.load = load <8 x i16>, <8 x i16>* %1, align 2
  %2 = zext <8 x i16> %wide.load to <8 x i32>
  %3 = getelementptr inbounds i16, i16* %y, i32 %index
  %4 = bitcast i16* %3 to <8 x i16>*
  %wide.load16 = load <8 x i16>, <8 x i16>* %4, align 2
  %5 = zext <8 x i16> %wide.load16 to <8 x i32>
  %6 = add nuw nsw <8 x i32> %2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %7 = add nuw nsw <8 x i32> %6, %5
  %8 = lshr <8 x i32> %7, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %9 = trunc <8 x i32> %8 to <8 x i16>
  %10 = getelementptr inbounds i16, i16* %z, i32 %index
  %11 = bitcast i16* %10 to <8 x i16>*
  store <8 x i16> %9, <8 x i16>* %11, align 2
  %index.next = add i32 %index, 8
  %12 = icmp eq i32 %index.next, 1024
  br i1 %12, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vrhadd_loop_s32(i32* nocapture readonly %x, i32* nocapture readonly %y, i32* noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vrhadd_loop_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #256
; CHECK-NEXT:  .LBB32_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
; CHECK-NEXT:    vrhadd.u32 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB32_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i32, i32* %x, i32 %index
  %1 = bitcast i32* %0 to <4 x i32>*
  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
  %2 = zext <4 x i32> %wide.load to <4 x i64>
  %3 = getelementptr inbounds i32, i32* %y, i32 %index
  %4 = bitcast i32* %3 to <4 x i32>*
  %wide.load16 = load <4 x i32>, <4 x i32>* %4, align 4
  %5 = zext <4 x i32> %wide.load16 to <4 x i64>
  %6 = add nuw nsw <4 x i64> %2, <i64 1, i64 1, i64 1, i64 1>
  %7 = add nuw nsw <4 x i64> %6, %5
  %8 = lshr <4 x i64> %7, <i64 1, i64 1, i64 1, i64 1>
  %9 = trunc <4 x i64> %8 to <4 x i32>
  %10 = getelementptr inbounds i32, i32* %z, i32 %index
  %11 = bitcast i32* %10 to <4 x i32>*
  store <4 x i32> %9, <4 x i32>* %11, align 4
  %index.next = add i32 %index, 4
  %12 = icmp eq i32 %index.next, 1024
  br i1 %12, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vrhadd_loop_u8(i8* nocapture readonly %x, i8* nocapture readonly %y, i8* noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vrhadd_loop_u8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #64
; CHECK-NEXT:  .LBB33_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r1], #16
; CHECK-NEXT:    vldrb.u8 q1, [r0], #16
; CHECK-NEXT:    vrhadd.u8 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB33_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i8, i8* %x, i32 %index
  %1 = bitcast i8* %0 to <16 x i8>*
  %wide.load = load <16 x i8>, <16 x i8>* %1, align 1
  %2 = zext <16 x i8> %wide.load to <16 x i16>
  %3 = getelementptr inbounds i8, i8* %y, i32 %index
  %4 = bitcast i8* %3 to <16 x i8>*
  %wide.load16 = load <16 x i8>, <16 x i8>* %4, align 1
  %5 = zext <16 x i8> %wide.load16 to <16 x i16>
  %6 = add nuw nsw <16 x i16> %2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %7 = add nuw nsw <16 x i16> %6, %5
  %8 = lshr <16 x i16> %7, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %9 = trunc <16 x i16> %8 to <16 x i8>
  %10 = getelementptr inbounds i8, i8* %z, i32 %index
  %11 = bitcast i8* %10 to <16 x i8>*
  store <16 x i8> %9, <16 x i8>* %11, align 1
  %index.next = add i32 %index, 16
  %12 = icmp eq i32 %index.next, 1024
  br i1 %12, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vrhadd_loop_u16(i16* nocapture readonly %x, i16* nocapture readonly %y, i16* noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vrhadd_loop_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #128
; CHECK-NEXT:  .LBB34_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
; CHECK-NEXT:    vrhadd.u16 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB34_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i16, i16* %x, i32 %index
  %1 = bitcast i16* %0 to <8 x i16>*
  %wide.load = load <8 x i16>, <8 x i16>* %1, align 2
  %2 = zext <8 x i16> %wide.load to <8 x i32>
  %3 = getelementptr inbounds i16, i16* %y, i32 %index
  %4 = bitcast i16* %3 to <8 x i16>*
  %wide.load16 = load <8 x i16>, <8 x i16>* %4, align 2
  %5 = zext <8 x i16> %wide.load16 to <8 x i32>
  %6 = add nuw nsw <8 x i32> %2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %7 = add nuw nsw <8 x i32> %6, %5
  %8 = lshr <8 x i32> %7, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %9 = trunc <8 x i32> %8 to <8 x i16>
  %10 = getelementptr inbounds i16, i16* %z, i32 %index
  %11 = bitcast i16* %10 to <8 x i16>*
  store <8 x i16> %9, <8 x i16>* %11, align 2
  %index.next = add i32 %index, 8
  %12 = icmp eq i32 %index.next, 1024
  br i1 %12, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vrhadd_loop_u32(i32* nocapture readonly %x, i32* nocapture readonly %y, i32* noalias nocapture %z, i32 %n) {
; CHECK-LABEL: vrhadd_loop_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #256
; CHECK-NEXT:  .LBB35_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
; CHECK-NEXT:    vrhadd.u32 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB35_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i32, i32* %x, i32 %index
  %1 = bitcast i32* %0 to <4 x i32>*
  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
  %2 = zext <4 x i32> %wide.load to <4 x i64>
  %3 = getelementptr inbounds i32, i32* %y, i32 %index
  %4 = bitcast i32* %3 to <4 x i32>*
  %wide.load16 = load <4 x i32>, <4 x i32>* %4, align 4
  %5 = zext <4 x i32> %wide.load16 to <4 x i64>
  %6 = add nuw nsw <4 x i64> %2, <i64 1, i64 1, i64 1, i64 1>
  %7 = add nuw nsw <4 x i64> %6, %5
  %8 = lshr <4 x i64> %7, <i64 1, i64 1, i64 1, i64 1>
  %9 = trunc <4 x i64> %8 to <4 x i32>
  %10 = getelementptr inbounds i32, i32* %z, i32 %index
  %11 = bitcast i32* %10 to <4 x i32>*
  store <4 x i32> %9, <4 x i32>* %11, align 4
  %index.next = add i32 %index, 4
  %12 = icmp eq i32 %index.next, 1024
  br i1 %12, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}