; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mattr=+sve -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=CHECK,SVE
; RUN: llc -mattr=+sve2 -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
; RUN: llc -mattr=+sme -force-streaming  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE

target triple = "aarch64-unknown-linux-gnu"

;
; sext i1 -> i32
;

; NOTE: Covers the scenario where a SIGN_EXTEND_INREG is required, whose inreg
; type's element type is not byte based and thus cannot be lowered directly to
; an SVE instruction.
define void @sext_v8i1_v8i32(<8 x i1> %a, ptr %out) {
; CHECK-LABEL: sext_v8i1_v8i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
; CHECK-NEXT:    uunpklo z0.h, z0.b
; CHECK-NEXT:    uunpklo z1.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    lsl z1.s, z1.s, #31
; CHECK-NEXT:    lsl z0.s, z0.s, #31
; CHECK-NEXT:    asr z1.s, z1.s, #31
; CHECK-NEXT:    asr z0.s, z0.s, #31
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v8i1_v8i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #80
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #15]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #22]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #13]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #11]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #9]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #44]
; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #42]
; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #34]
; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #36]
; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #38]
; NONEON-NOSVE-NEXT:    sbfx w12, w12, #0, #1
; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
; NONEON-NOSVE-NEXT:    stp w10, w8, [sp, #72]
; NONEON-NOSVE-NEXT:    sbfx w8, w14, #0, #1
; NONEON-NOSVE-NEXT:    sbfx w10, w15, #0, #1
; NONEON-NOSVE-NEXT:    stp w8, w12, [sp, #64]
; NONEON-NOSVE-NEXT:    sbfx w12, w13, #0, #1
; NONEON-NOSVE-NEXT:    sbfx w8, w11, #0, #1
; NONEON-NOSVE-NEXT:    stp w12, w10, [sp, #56]
; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #48]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #80
; NONEON-NOSVE-NEXT:    ret
  %b = sext <8 x i1> %a to <8 x i32>
  store <8 x i32> %b, ptr %out
  ret void
}

;
; sext i3 -> i64
;

; NOTE: Covers the scenario where a SIGN_EXTEND_INREG is required, whose inreg
; type's element type is not power-of-2 based and thus cannot be lowered
; directly to an SVE instruction.
define void @sext_v4i3_v4i64(<4 x i3> %a, ptr %out) {
; CHECK-LABEL: sext_v4i3_v4i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    uunpklo z1.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.d, z0.s
; CHECK-NEXT:    lsl z1.d, z1.d, #61
; CHECK-NEXT:    lsl z0.d, z0.d, #61
; CHECK-NEXT:    asr z1.d, z1.d, #61
; CHECK-NEXT:    asr z0.d, z0.d, #61
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v4i3_v4i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
; NONEON-NOSVE-NEXT:    str q0, [sp, #-64]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp w9, w8, [sp, #24]
; NONEON-NOSVE-NEXT:    ldp w11, w10, [sp, #16]
; NONEON-NOSVE-NEXT:    sbfx x8, x8, #0, #3
; NONEON-NOSVE-NEXT:    sbfx x9, x9, #0, #3
; NONEON-NOSVE-NEXT:    sbfx x10, x10, #0, #3
; NONEON-NOSVE-NEXT:    stp x9, x8, [sp, #48]
; NONEON-NOSVE-NEXT:    sbfx x8, x11, #0, #3
; NONEON-NOSVE-NEXT:    stp x8, x10, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #64
; NONEON-NOSVE-NEXT:    ret
  %b = sext <4 x i3> %a to <4 x i64>
  store <4 x i64> %b, ptr %out
  ret void
}

;
; sext i8 -> i16
;

define void @sext_v16i8_v16i16(<16 x i8> %a, ptr %out) {
; CHECK-LABEL: sext_v16i8_v16i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    sunpklo z1.h, z0.b
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z0.h, z0.b
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v16i8_v16i16:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    str q0, [sp, #-64]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #31]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #30]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #29]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #28]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #27]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #25]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #24]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #23]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #22]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #21]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #20]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #19]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #18]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #17]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #16]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #64
; NONEON-NOSVE-NEXT:    ret
  %b = sext <16 x i8> %a to <16 x i16>
  store <16 x i16>%b, ptr %out
  ret void
}

; NOTE: Extra 'add' is to prevent the extend being combined with the load.
define void @sext_v32i8_v32i16(ptr %in, ptr %out) {
; CHECK-LABEL: sext_v32i8_v32i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldp q1, q0, [x0]
; CHECK-NEXT:    add z0.b, z0.b, z0.b
; CHECK-NEXT:    add z1.b, z1.b, z1.b
; CHECK-NEXT:    sunpklo z2.h, z0.b
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z3.h, z1.b
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    sunpklo z0.h, z0.b
; CHECK-NEXT:    sunpklo z1.h, z1.b
; CHECK-NEXT:    stp q2, q0, [x1, #32]
; CHECK-NEXT:    stp q3, q1, [x1]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v32i8_v32i16:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #272
; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #176] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #192] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #208] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #224] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #240] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #256] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 272
; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88
; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #41]
; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #28]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #38]
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #17]
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #22]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
; NONEON-NOSVE-NEXT:    add w8, w29, w29
; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #36]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
; NONEON-NOSVE-NEXT:    add w8, w27, w27
; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #31]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
; NONEON-NOSVE-NEXT:    add w8, w25, w25
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
; NONEON-NOSVE-NEXT:    add w8, w23, w23
; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #34]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
; NONEON-NOSVE-NEXT:    add w8, w21, w21
; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #29]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
; NONEON-NOSVE-NEXT:    add w8, w19, w19
; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #23]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]
; NONEON-NOSVE-NEXT:    add w9, w28, w28
; NONEON-NOSVE-NEXT:    add w18, w16, w16
; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
; NONEON-NOSVE-NEXT:    add w8, w6, w6
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #27]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]
; NONEON-NOSVE-NEXT:    add w9, w26, w26
; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
; NONEON-NOSVE-NEXT:    add w8, w4, w4
; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #25]
; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]
; NONEON-NOSVE-NEXT:    add w9, w24, w24
; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
; NONEON-NOSVE-NEXT:    add w8, w2, w2
; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
; NONEON-NOSVE-NEXT:    add w17, w17, w17
; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]
; NONEON-NOSVE-NEXT:    add w9, w22, w22
; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
; NONEON-NOSVE-NEXT:    add w8, w16, w16
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
; NONEON-NOSVE-NEXT:    strb w17, [sp, #63]
; NONEON-NOSVE-NEXT:    add w17, w30, w30
; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]
; NONEON-NOSVE-NEXT:    add w9, w20, w20
; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #39]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
; NONEON-NOSVE-NEXT:    add w8, w14, w14
; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #37]
; NONEON-NOSVE-NEXT:    strb w18, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #35]
; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #33]
; NONEON-NOSVE-NEXT:    strb w17, [sp, #61]
; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #47]
; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #45]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]
; NONEON-NOSVE-NEXT:    add w9, w7, w7
; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #43]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
; NONEON-NOSVE-NEXT:    add w8, w12, w12
; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
; NONEON-NOSVE-NEXT:    add w8, w10, w10
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #79]
; NONEON-NOSVE-NEXT:    add w9, w5, w5
; NONEON-NOSVE-NEXT:    strb w9, [sp, #77]
; NONEON-NOSVE-NEXT:    add w9, w3, w3
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #95]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #75]
; NONEON-NOSVE-NEXT:    add w9, w0, w0
; NONEON-NOSVE-NEXT:    strh w8, [sp, #142]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #94]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #73]
; NONEON-NOSVE-NEXT:    add w9, w15, w15
; NONEON-NOSVE-NEXT:    strh w8, [sp, #140]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #93]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #71]
; NONEON-NOSVE-NEXT:    add w9, w13, w13
; NONEON-NOSVE-NEXT:    strh w8, [sp, #138]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #92]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #69]
; NONEON-NOSVE-NEXT:    add w9, w11, w11
; NONEON-NOSVE-NEXT:    strh w8, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #91]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #67]
; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #134]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #90]
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #256] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #132]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #89]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #240] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #130]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #88]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #87]
; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #224] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #126]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #86]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #208] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #124]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #85]
; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #192] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #122]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #84]
; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #176] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #83]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #118]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #82]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #116]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #81]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #114]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #80]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #111]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #112]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #174]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #110]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #172]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #109]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #170]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #108]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #168]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #107]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #166]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #106]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #164]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #105]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #162]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #104]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #160]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #103]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #158]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #102]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #156]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #101]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #154]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #100]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #99]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #150]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #98]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #148]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #97]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #146]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #96]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #144]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #144]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
; NONEON-NOSVE-NEXT:    add sp, sp, #272
; NONEON-NOSVE-NEXT:    ret
  %a = load <32 x i8>, ptr %in
  %b = add <32 x i8> %a, %a
  %c = sext <32 x i8> %b to <32 x i16>
  store <32 x i16> %c, ptr %out
  ret void
}

;
; sext i8 -> i32
;

define void @sext_v8i8_v8i32(<8 x i8> %a, ptr %out) {
; CHECK-LABEL: sext_v8i8_v8i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
; CHECK-NEXT:    sunpklo z0.h, z0.b
; CHECK-NEXT:    sunpklo z1.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z0.s, z0.h
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v8i8_v8i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #80
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #11]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #10]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #9]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #8]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #15]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #22]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #14]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #13]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #12]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #44]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #42]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #40]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #38]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #36]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #34]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #32]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #80
; NONEON-NOSVE-NEXT:    ret
  %b = sext <8 x i8> %a to <8 x i32>
  store <8 x i32>%b, ptr %out
  ret void
}

define void @sext_v16i8_v16i32(<16 x i8> %a, ptr %out) {
; CHECK-LABEL: sext_v16i8_v16i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    sunpklo z1.h, z0.b
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z0.h, z0.b
; CHECK-NEXT:    sunpklo z2.s, z1.h
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    sunpklo z1.s, z1.h
; CHECK-NEXT:    sunpklo z3.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z0.s, z0.h
; CHECK-NEXT:    stp q2, q1, [x0]
; CHECK-NEXT:    stp q3, q0, [x0, #32]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v16i8_v16i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    str q0, [sp, #-160]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #27]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #25]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #24]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #31]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #30]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #29]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #28]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #19]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #18]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #17]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #94]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #23]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #22]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #21]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #20]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #92]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #90]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #88]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #86]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #84]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #82]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #80]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #78]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #76]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #128]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #74]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #72]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #70]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #68]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #66]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #64]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
; NONEON-NOSVE-NEXT:    add sp, sp, #160
; NONEON-NOSVE-NEXT:    ret
  %b = sext <16 x i8> %a to <16 x i32>
  store <16 x i32> %b, ptr %out
  ret void
}

define void @sext_v32i8_v32i32(ptr %in, ptr %out) {
; CHECK-LABEL: sext_v32i8_v32i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldp q1, q0, [x0]
; CHECK-NEXT:    add z0.b, z0.b, z0.b
; CHECK-NEXT:    add z1.b, z1.b, z1.b
; CHECK-NEXT:    sunpklo z2.h, z0.b
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z3.h, z1.b
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    sunpklo z0.h, z0.b
; CHECK-NEXT:    sunpklo z1.h, z1.b
; CHECK-NEXT:    sunpklo z4.s, z2.h
; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
; CHECK-NEXT:    sunpklo z5.s, z3.h
; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
; CHECK-NEXT:    sunpklo z6.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z2.s, z2.h
; CHECK-NEXT:    sunpklo z7.s, z1.h
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    sunpklo z3.s, z3.h
; CHECK-NEXT:    sunpklo z0.s, z0.h
; CHECK-NEXT:    sunpklo z1.s, z1.h
; CHECK-NEXT:    stp q4, q2, [x1, #64]
; CHECK-NEXT:    stp q5, q3, [x1]
; CHECK-NEXT:    stp q6, q0, [x1, #96]
; CHECK-NEXT:    stp q7, q1, [x1, #32]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v32i8_v32i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #464
; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #368] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #384] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #400] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #416] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #432] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #448] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 464
; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88
; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #41]
; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #28]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #38]
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #17]
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #22]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
; NONEON-NOSVE-NEXT:    add w8, w29, w29
; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #36]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
; NONEON-NOSVE-NEXT:    add w8, w27, w27
; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #31]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
; NONEON-NOSVE-NEXT:    add w8, w25, w25
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
; NONEON-NOSVE-NEXT:    add w8, w23, w23
; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #34]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
; NONEON-NOSVE-NEXT:    add w8, w21, w21
; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #29]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
; NONEON-NOSVE-NEXT:    add w8, w19, w19
; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #23]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]
; NONEON-NOSVE-NEXT:    add w9, w28, w28
; NONEON-NOSVE-NEXT:    add w18, w16, w16
; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
; NONEON-NOSVE-NEXT:    add w8, w6, w6
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #27]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]
; NONEON-NOSVE-NEXT:    add w9, w26, w26
; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
; NONEON-NOSVE-NEXT:    add w8, w4, w4
; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #25]
; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]
; NONEON-NOSVE-NEXT:    add w9, w24, w24
; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
; NONEON-NOSVE-NEXT:    add w8, w2, w2
; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
; NONEON-NOSVE-NEXT:    add w17, w17, w17
; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]
; NONEON-NOSVE-NEXT:    add w9, w22, w22
; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
; NONEON-NOSVE-NEXT:    add w8, w16, w16
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
; NONEON-NOSVE-NEXT:    strb w17, [sp, #63]
; NONEON-NOSVE-NEXT:    add w17, w30, w30
; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]
; NONEON-NOSVE-NEXT:    add w9, w20, w20
; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #39]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
; NONEON-NOSVE-NEXT:    add w8, w14, w14
; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #37]
; NONEON-NOSVE-NEXT:    strb w18, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #35]
; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #33]
; NONEON-NOSVE-NEXT:    strb w17, [sp, #61]
; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #47]
; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #45]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]
; NONEON-NOSVE-NEXT:    add w9, w7, w7
; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #43]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
; NONEON-NOSVE-NEXT:    add w8, w12, w12
; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
; NONEON-NOSVE-NEXT:    add w8, w10, w10
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #79]
; NONEON-NOSVE-NEXT:    add w9, w5, w5
; NONEON-NOSVE-NEXT:    strb w9, [sp, #77]
; NONEON-NOSVE-NEXT:    add w9, w3, w3
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #91]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #75]
; NONEON-NOSVE-NEXT:    add w9, w0, w0
; NONEON-NOSVE-NEXT:    strh w8, [sp, #134]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #90]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #73]
; NONEON-NOSVE-NEXT:    add w9, w15, w15
; NONEON-NOSVE-NEXT:    strh w8, [sp, #132]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #89]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #71]
; NONEON-NOSVE-NEXT:    add w9, w13, w13
; NONEON-NOSVE-NEXT:    strh w8, [sp, #130]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #88]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #69]
; NONEON-NOSVE-NEXT:    add w9, w11, w11
; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #95]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #67]
; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #142]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #94]
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #448] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #140]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #93]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #432] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #138]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #92]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #83]
; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #416] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #118]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #82]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #128]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #116]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #81]
; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #400] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #114]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #80]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #192]
; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #384] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #87]
; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #368] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #126]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #86]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #124]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #85]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #122]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #84]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #107]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #112]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #166]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #106]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #164]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #105]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #176]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #162]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #182]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #160]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #111]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #174]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #110]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #172]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #109]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #170]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #108]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #168]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #99]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #160]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #150]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #98]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #148]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #97]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #224]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #146]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #96]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #144]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #103]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #158]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #102]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #156]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #101]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #154]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #100]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #198]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #144]
; NONEON-NOSVE-NEXT:    str w8, [sp, #284]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #196]
; NONEON-NOSVE-NEXT:    str w8, [sp, #280]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #194]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #208]
; NONEON-NOSVE-NEXT:    str w8, [sp, #276]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #192]
; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #206]
; NONEON-NOSVE-NEXT:    str w8, [sp, #300]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #204]
; NONEON-NOSVE-NEXT:    str w8, [sp, #296]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #202]
; NONEON-NOSVE-NEXT:    str w8, [sp, #292]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #200]
; NONEON-NOSVE-NEXT:    str w8, [sp, #288]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #180]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #272]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #248]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #178]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #176]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #240]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #190]
; NONEON-NOSVE-NEXT:    str w8, [sp, #268]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #188]
; NONEON-NOSVE-NEXT:    str w8, [sp, #264]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #186]
; NONEON-NOSVE-NEXT:    str w8, [sp, #260]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #184]
; NONEON-NOSVE-NEXT:    str w8, [sp, #256]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #230]
; NONEON-NOSVE-NEXT:    ldp q3, q4, [sp, #240]
; NONEON-NOSVE-NEXT:    str w8, [sp, #348]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #228]
; NONEON-NOSVE-NEXT:    str w8, [sp, #344]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #226]
; NONEON-NOSVE-NEXT:    str w8, [sp, #340]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #224]
; NONEON-NOSVE-NEXT:    str w8, [sp, #336]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #238]
; NONEON-NOSVE-NEXT:    str w8, [sp, #364]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #236]
; NONEON-NOSVE-NEXT:    str w8, [sp, #360]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #234]
; NONEON-NOSVE-NEXT:    str w8, [sp, #356]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #232]
; NONEON-NOSVE-NEXT:    str w8, [sp, #352]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #214]
; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #336]
; NONEON-NOSVE-NEXT:    str w8, [sp, #316]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #212]
; NONEON-NOSVE-NEXT:    str w8, [sp, #312]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #210]
; NONEON-NOSVE-NEXT:    str w8, [sp, #308]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #208]
; NONEON-NOSVE-NEXT:    str w8, [sp, #304]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #222]
; NONEON-NOSVE-NEXT:    str w8, [sp, #332]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #220]
; NONEON-NOSVE-NEXT:    str w8, [sp, #328]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #218]
; NONEON-NOSVE-NEXT:    str w8, [sp, #324]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #216]
; NONEON-NOSVE-NEXT:    str w8, [sp, #320]
; NONEON-NOSVE-NEXT:    ldp q5, q2, [sp, #304]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
; NONEON-NOSVE-NEXT:    stp q3, q4, [x1, #32]
; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #64]
; NONEON-NOSVE-NEXT:    stp q5, q2, [x1, #96]
; NONEON-NOSVE-NEXT:    add sp, sp, #464
; NONEON-NOSVE-NEXT:    ret
  %a = load <32 x i8>, ptr %in
  %b = add <32 x i8> %a, %a
  %c = sext <32 x i8> %b to <32 x i32>
  store <32 x i32> %c, ptr %out
  ret void
}

;
; sext i8 -> i64
;

; NOTE: v4i8 is an unpacked typed stored within a v4i16 container. The sign
; extend is a two step process where the container is any_extend'd with the
; result feeding an inreg sign extend.
define void @sext_v4i8_v4i64(<4 x i8> %a, ptr %out) {
; CHECK-LABEL: sext_v4i8_v4i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
; CHECK-NEXT:    ptrue p0.d, vl2
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    uunpklo z1.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.d, z0.s
; CHECK-NEXT:    sxtb z1.d, p0/m, z1.d
; CHECK-NEXT:    sxtb z0.d, p0/m, z0.d
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v4i8_v4i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    ushll v0.4s, v0.4h, #0
; NONEON-NOSVE-NEXT:    str q0, [sp, #-64]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrsb x8, [sp, #28]
; NONEON-NOSVE-NEXT:    ldrsb x9, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrsb x10, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrsb x11, [sp, #20]
; NONEON-NOSVE-NEXT:    stp x9, x8, [sp, #48]
; NONEON-NOSVE-NEXT:    stp x10, x11, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #64
; NONEON-NOSVE-NEXT:    ret
  %b = sext <4 x i8> %a to <4 x i64>
  store <4 x i64>%b, ptr %out
  ret void
}

define void @sext_v8i8_v8i64(<8 x i8> %a, ptr %out) {
; CHECK-LABEL: sext_v8i8_v8i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
; CHECK-NEXT:    sunpklo z0.h, z0.b
; CHECK-NEXT:    sunpklo z1.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z0.s, z0.h
; CHECK-NEXT:    sunpklo z2.d, z1.s
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    sunpklo z3.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z1.d, z1.s
; CHECK-NEXT:    sunpklo z0.d, z0.s
; CHECK-NEXT:    stp q2, q1, [x0]
; CHECK-NEXT:    stp q3, q0, [x0, #32]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v8i8_v8i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #176
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 176
; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
; NONEON-NOSVE-NEXT:    add x8, sp, #144
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #11]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #10]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #28]
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #9]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #8]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #15]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #22]
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #14]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #13]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #12]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #42]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #40]
; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #64]
; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #44]
; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #72]
; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #34]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #64]
; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #38]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #36]
; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #56]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #48]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #80]
; NONEON-NOSVE-NEXT:    ldpsw x9, x10, [sp, #96]
; NONEON-NOSVE-NEXT:    stp x9, x10, [sp, #144]
; NONEON-NOSVE-NEXT:    ldpsw x9, x10, [sp, #104]
; NONEON-NOSVE-NEXT:    stp x9, x10, [sp, #160]
; NONEON-NOSVE-NEXT:    ldpsw x9, x10, [sp, #80]
; NONEON-NOSVE-NEXT:    stp x9, x10, [sp, #112]
; NONEON-NOSVE-NEXT:    ldpsw x9, x10, [sp, #88]
; NONEON-NOSVE-NEXT:    stp x9, x10, [sp, #128]
; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #112]
; NONEON-NOSVE-NEXT:    ldp q2, q3, [x8]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #176
; NONEON-NOSVE-NEXT:    ret
  %b = sext <8 x i8> %a to <8 x i64>
  store <8 x i64>%b, ptr %out
  ret void
}

define void @sext_v16i8_v16i64(<16 x i8> %a, ptr %out) {
; CHECK-LABEL: sext_v16i8_v16i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    sunpklo z1.h, z0.b
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z0.h, z0.b
; CHECK-NEXT:    sunpklo z2.s, z1.h
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    sunpklo z1.s, z1.h
; CHECK-NEXT:    sunpklo z3.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z4.d, z2.s
; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
; CHECK-NEXT:    sunpklo z0.s, z0.h
; CHECK-NEXT:    sunpklo z6.d, z1.s
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    sunpklo z5.d, z3.s
; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
; CHECK-NEXT:    sunpklo z2.d, z2.s
; CHECK-NEXT:    sunpklo z1.d, z1.s
; CHECK-NEXT:    sunpklo z7.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z3.d, z3.s
; CHECK-NEXT:    stp q4, q2, [x0]
; CHECK-NEXT:    sunpklo z0.d, z0.s
; CHECK-NEXT:    stp q6, q1, [x0, #32]
; CHECK-NEXT:    stp q5, q3, [x0, #64]
; CHECK-NEXT:    stp q7, q0, [x0, #96]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v16i8_v16i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #368
; NONEON-NOSVE-NEXT:    str x29, [sp, #352] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 368
; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
; NONEON-NOSVE-NEXT:    str q0, [sp]
; NONEON-NOSVE-NEXT:    ldr x29, [sp, #352] // 8-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #35]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #70]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #34]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #68]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #33]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #66]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #32]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #64]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #39]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #38]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #37]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #36]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #27]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #56]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #25]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #88]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #98]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #31]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #30]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #29]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #28]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #40]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #102]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #100]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #160]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #90]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #88]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #72]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #152]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #94]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #92]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #82]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #80]
; NONEON-NOSVE-NEXT:    str d0, [sp, #360]
; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp, #136]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #86]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #84]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #74]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #72]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #120]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #78]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #76]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
; NONEON-NOSVE-NEXT:    stp d0, d2, [sp, #192]
; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp, #104]
; NONEON-NOSVE-NEXT:    str d2, [sp, #168]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #176]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #216]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #320]
; NONEON-NOSVE-NEXT:    ldrsw x9, [sp, #364]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #360]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #336]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #200]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #320]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #288]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #208]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #304]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #184]
; NONEON-NOSVE-NEXT:    ldp q3, q4, [sp, #288]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #256]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #192]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #272]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #168]
; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #256]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #224]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #176]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #240]
; NONEON-NOSVE-NEXT:    ldp q5, q2, [sp, #224]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    stp q3, q4, [x0, #32]
; NONEON-NOSVE-NEXT:    stp q6, q7, [x0, #64]
; NONEON-NOSVE-NEXT:    stp q5, q2, [x0, #96]
; NONEON-NOSVE-NEXT:    add sp, sp, #368
; NONEON-NOSVE-NEXT:    ret
  %b = sext <16 x i8> %a to <16 x i64>
  store <16 x i64> %b, ptr %out
  ret void
}

define void @sext_v32i8_v32i64(ptr %in, ptr %out) {
; CHECK-LABEL: sext_v32i8_v32i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldp q1, q0, [x0]
; CHECK-NEXT:    add z0.b, z0.b, z0.b
; CHECK-NEXT:    add z1.b, z1.b, z1.b
; CHECK-NEXT:    mov z2.d, z0.d
; CHECK-NEXT:    sunpklo z0.h, z0.b
; CHECK-NEXT:    mov z3.d, z1.d
; CHECK-NEXT:    sunpklo z1.h, z1.b
; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
; CHECK-NEXT:    sunpklo z4.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z5.s, z1.h
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    sunpklo z2.h, z2.b
; CHECK-NEXT:    sunpklo z3.h, z3.b
; CHECK-NEXT:    sunpklo z0.s, z0.h
; CHECK-NEXT:    sunpklo z16.d, z4.s
; CHECK-NEXT:    ext z4.b, z4.b, z4.b, #8
; CHECK-NEXT:    sunpklo z1.s, z1.h
; CHECK-NEXT:    sunpklo z17.d, z5.s
; CHECK-NEXT:    ext z5.b, z5.b, z5.b, #8
; CHECK-NEXT:    sunpklo z6.s, z2.h
; CHECK-NEXT:    sunpklo z7.s, z3.h
; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
; CHECK-NEXT:    sunpklo z4.d, z4.s
; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
; CHECK-NEXT:    sunpklo z19.d, z0.s
; CHECK-NEXT:    sunpklo z5.d, z5.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z2.s, z2.h
; CHECK-NEXT:    sunpklo z18.d, z6.s
; CHECK-NEXT:    ext z6.b, z6.b, z6.b, #8
; CHECK-NEXT:    sunpklo z3.s, z3.h
; CHECK-NEXT:    stp q16, q4, [x1, #128]
; CHECK-NEXT:    mov z16.d, z7.d
; CHECK-NEXT:    sunpklo z0.d, z0.s
; CHECK-NEXT:    stp q17, q5, [x1]
; CHECK-NEXT:    sunpklo z5.d, z7.s
; CHECK-NEXT:    sunpklo z4.d, z6.s
; CHECK-NEXT:    mov z6.d, z1.d
; CHECK-NEXT:    ext z16.b, z16.b, z7.b, #8
; CHECK-NEXT:    mov z7.d, z2.d
; CHECK-NEXT:    stp q19, q0, [x1, #160]
; CHECK-NEXT:    sunpklo z0.d, z2.s
; CHECK-NEXT:    ext z6.b, z6.b, z1.b, #8
; CHECK-NEXT:    sunpklo z1.d, z1.s
; CHECK-NEXT:    stp q18, q4, [x1, #192]
; CHECK-NEXT:    mov z4.d, z3.d
; CHECK-NEXT:    ext z7.b, z7.b, z2.b, #8
; CHECK-NEXT:    sunpklo z16.d, z16.s
; CHECK-NEXT:    sunpklo z6.d, z6.s
; CHECK-NEXT:    ext z4.b, z4.b, z3.b, #8
; CHECK-NEXT:    sunpklo z2.d, z7.s
; CHECK-NEXT:    sunpklo z3.d, z3.s
; CHECK-NEXT:    stp q5, q16, [x1, #64]
; CHECK-NEXT:    stp q1, q6, [x1, #32]
; CHECK-NEXT:    sunpklo z1.d, z4.s
; CHECK-NEXT:    stp q0, q2, [x1, #224]
; CHECK-NEXT:    stp q3, q1, [x1, #96]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v32i8_v32i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #-96]! // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #16] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #32] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #48] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #64] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    sub sp, sp, #752
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 848
; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88
; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #41]
; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #28]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #38]
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #17]
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #22]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
; NONEON-NOSVE-NEXT:    add w8, w29, w29
; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #36]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
; NONEON-NOSVE-NEXT:    add w8, w27, w27
; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #31]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
; NONEON-NOSVE-NEXT:    add w8, w25, w25
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
; NONEON-NOSVE-NEXT:    add w8, w23, w23
; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #34]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
; NONEON-NOSVE-NEXT:    add w8, w21, w21
; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #29]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
; NONEON-NOSVE-NEXT:    add w8, w19, w19
; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #23]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]
; NONEON-NOSVE-NEXT:    add w9, w28, w28
; NONEON-NOSVE-NEXT:    add w18, w16, w16
; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
; NONEON-NOSVE-NEXT:    add w8, w6, w6
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #27]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]
; NONEON-NOSVE-NEXT:    add w9, w26, w26
; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
; NONEON-NOSVE-NEXT:    add w8, w4, w4
; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #25]
; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]
; NONEON-NOSVE-NEXT:    add w9, w24, w24
; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
; NONEON-NOSVE-NEXT:    add w8, w2, w2
; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
; NONEON-NOSVE-NEXT:    add w17, w17, w17
; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]
; NONEON-NOSVE-NEXT:    add w9, w22, w22
; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
; NONEON-NOSVE-NEXT:    add w8, w16, w16
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
; NONEON-NOSVE-NEXT:    strb w17, [sp, #63]
; NONEON-NOSVE-NEXT:    add w17, w30, w30
; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]
; NONEON-NOSVE-NEXT:    add w9, w20, w20
; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #39]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
; NONEON-NOSVE-NEXT:    add w8, w14, w14
; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #37]
; NONEON-NOSVE-NEXT:    strb w18, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #35]
; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #33]
; NONEON-NOSVE-NEXT:    strb w17, [sp, #61]
; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #47]
; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #45]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]
; NONEON-NOSVE-NEXT:    add w9, w7, w7
; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #43]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
; NONEON-NOSVE-NEXT:    add w8, w12, w12
; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
; NONEON-NOSVE-NEXT:    add w8, w10, w10
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #79]
; NONEON-NOSVE-NEXT:    add w9, w5, w5
; NONEON-NOSVE-NEXT:    strb w9, [sp, #77]
; NONEON-NOSVE-NEXT:    add w9, w3, w3
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #91]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #75]
; NONEON-NOSVE-NEXT:    add w9, w0, w0
; NONEON-NOSVE-NEXT:    strh w8, [sp, #134]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #90]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #73]
; NONEON-NOSVE-NEXT:    add w9, w15, w15
; NONEON-NOSVE-NEXT:    strh w8, [sp, #132]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #89]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #71]
; NONEON-NOSVE-NEXT:    add w9, w13, w13
; NONEON-NOSVE-NEXT:    strh w8, [sp, #130]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #88]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #69]
; NONEON-NOSVE-NEXT:    add w9, w11, w11
; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #95]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #67]
; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #142]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #94]
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    strh w8, [sp, #140]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #93]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #138]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #92]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #83]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #118]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #82]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #128]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #116]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #81]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #114]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #80]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #192]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #87]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #126]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #86]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #124]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #85]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #122]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #84]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #107]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #112]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #166]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #106]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #164]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #105]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #176]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #162]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #178]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #160]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #111]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #174]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #110]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #172]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #109]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #170]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #108]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #168]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #99]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #160]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #150]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #98]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #148]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #97]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #224]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #146]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #96]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #144]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #103]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #158]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #102]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #156]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #101]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #154]
; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #100]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #194]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #144]
; NONEON-NOSVE-NEXT:    str w8, [sp, #276]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #192]
; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #198]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #208]
; NONEON-NOSVE-NEXT:    str w8, [sp, #284]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #196]
; NONEON-NOSVE-NEXT:    str w8, [sp, #280]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #202]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #272]
; NONEON-NOSVE-NEXT:    str w8, [sp, #292]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #200]
; NONEON-NOSVE-NEXT:    str w8, [sp, #288]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #206]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #400]
; NONEON-NOSVE-NEXT:    str w8, [sp, #300]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #204]
; NONEON-NOSVE-NEXT:    str w8, [sp, #296]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #176]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #288]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #240]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #182]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #180]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #248]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #186]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #416]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #240]
; NONEON-NOSVE-NEXT:    str w8, [sp, #260]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #184]
; NONEON-NOSVE-NEXT:    str w8, [sp, #256]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #190]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #368]
; NONEON-NOSVE-NEXT:    str w8, [sp, #268]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #188]
; NONEON-NOSVE-NEXT:    ldrsw x9, [sp, #372]
; NONEON-NOSVE-NEXT:    str w8, [sp, #264]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #226]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #256]
; NONEON-NOSVE-NEXT:    str w8, [sp, #340]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #224]
; NONEON-NOSVE-NEXT:    str w8, [sp, #336]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #230]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #384]
; NONEON-NOSVE-NEXT:    str w8, [sp, #348]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #228]
; NONEON-NOSVE-NEXT:    str w8, [sp, #344]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #234]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #336]
; NONEON-NOSVE-NEXT:    str w8, [sp, #356]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #232]
; NONEON-NOSVE-NEXT:    str w8, [sp, #352]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #238]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #464]
; NONEON-NOSVE-NEXT:    str w8, [sp, #364]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #236]
; NONEON-NOSVE-NEXT:    str w8, [sp, #360]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #210]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #352]
; NONEON-NOSVE-NEXT:    str w8, [sp, #308]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #208]
; NONEON-NOSVE-NEXT:    str w8, [sp, #304]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #214]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #480]
; NONEON-NOSVE-NEXT:    str w8, [sp, #316]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #212]
; NONEON-NOSVE-NEXT:    str w8, [sp, #312]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #218]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #304]
; NONEON-NOSVE-NEXT:    str w8, [sp, #324]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #216]
; NONEON-NOSVE-NEXT:    str w8, [sp, #320]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #222]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #432]
; NONEON-NOSVE-NEXT:    str w8, [sp, #332]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #220]
; NONEON-NOSVE-NEXT:    str w8, [sp, #328]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #404]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #320]
; NONEON-NOSVE-NEXT:    str x8, [sp, #568]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #400]
; NONEON-NOSVE-NEXT:    str x8, [sp, #560]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #412]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #448]
; NONEON-NOSVE-NEXT:    str x8, [sp, #584]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #408]
; NONEON-NOSVE-NEXT:    str x8, [sp, #576]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #420]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #560]
; NONEON-NOSVE-NEXT:    str x8, [sp, #600]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #416]
; NONEON-NOSVE-NEXT:    str x8, [sp, #592]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #428]
; NONEON-NOSVE-NEXT:    str x8, [sp, #616]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #424]
; NONEON-NOSVE-NEXT:    str x8, [sp, #608]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #368]
; NONEON-NOSVE-NEXT:    ldp q2, q3, [sp, #592]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #496]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #380]
; NONEON-NOSVE-NEXT:    str x8, [sp, #520]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #376]
; NONEON-NOSVE-NEXT:    str x8, [sp, #512]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #388]
; NONEON-NOSVE-NEXT:    ldp q4, q5, [sp, #496]
; NONEON-NOSVE-NEXT:    str x8, [sp, #536]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #384]
; NONEON-NOSVE-NEXT:    str x8, [sp, #528]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #396]
; NONEON-NOSVE-NEXT:    str x8, [sp, #552]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #392]
; NONEON-NOSVE-NEXT:    str x8, [sp, #544]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #468]
; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #528]
; NONEON-NOSVE-NEXT:    str x8, [sp, #696]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #464]
; NONEON-NOSVE-NEXT:    str x8, [sp, #688]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #476]
; NONEON-NOSVE-NEXT:    str x8, [sp, #712]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #472]
; NONEON-NOSVE-NEXT:    str x8, [sp, #704]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #484]
; NONEON-NOSVE-NEXT:    ldp q16, q17, [sp, #688]
; NONEON-NOSVE-NEXT:    str x8, [sp, #728]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #480]
; NONEON-NOSVE-NEXT:    str x8, [sp, #720]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #492]
; NONEON-NOSVE-NEXT:    str x8, [sp, #744]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #488]
; NONEON-NOSVE-NEXT:    str x8, [sp, #736]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #436]
; NONEON-NOSVE-NEXT:    ldp q19, q20, [sp, #720]
; NONEON-NOSVE-NEXT:    str x8, [sp, #632]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #432]
; NONEON-NOSVE-NEXT:    str x8, [sp, #624]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #444]
; NONEON-NOSVE-NEXT:    str x8, [sp, #648]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #440]
; NONEON-NOSVE-NEXT:    str x8, [sp, #640]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #452]
; NONEON-NOSVE-NEXT:    ldp q22, q23, [sp, #624]
; NONEON-NOSVE-NEXT:    str x8, [sp, #664]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #448]
; NONEON-NOSVE-NEXT:    str x8, [sp, #656]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #460]
; NONEON-NOSVE-NEXT:    str x8, [sp, #680]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #456]
; NONEON-NOSVE-NEXT:    str x8, [sp, #672]
; NONEON-NOSVE-NEXT:    ldp q21, q18, [sp, #656]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x1, #32]
; NONEON-NOSVE-NEXT:    stp q4, q5, [x1, #64]
; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #96]
; NONEON-NOSVE-NEXT:    stp q16, q17, [x1, #128]
; NONEON-NOSVE-NEXT:    stp q19, q20, [x1, #160]
; NONEON-NOSVE-NEXT:    stp q22, q23, [x1, #192]
; NONEON-NOSVE-NEXT:    stp q21, q18, [x1, #224]
; NONEON-NOSVE-NEXT:    add sp, sp, #752
; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #64] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #48] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #32] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #16] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp], #96 // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ret
  %a = load <32 x i8>, ptr %in
  %b = add <32 x i8> %a, %a
  %c = sext <32 x i8> %b to <32 x i64>
  store <32 x i64> %c, ptr %out
  ret void
}

;
; sext i16 -> i32
;

define void @sext_v8i16_v8i32(<8 x i16> %a, ptr %out) {
; CHECK-LABEL: sext_v8i16_v8i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    sunpklo z1.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z0.s, z0.h
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v8i16_v8i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    str q0, [sp, #-64]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #28]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #22]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #20]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #16]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #64
; NONEON-NOSVE-NEXT:    ret
  %b = sext <8 x i16> %a to <8 x i32>
  store <8 x i32>%b, ptr %out
  ret void
}

define void @sext_v16i16_v16i32(ptr %in, ptr %out) {
; CHECK-LABEL: sext_v16i16_v16i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldp q1, q0, [x0]
; CHECK-NEXT:    add z0.h, z0.h, z0.h
; CHECK-NEXT:    add z1.h, z1.h, z1.h
; CHECK-NEXT:    sunpklo z2.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z3.s, z1.h
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    sunpklo z0.s, z0.h
; CHECK-NEXT:    sunpklo z1.s, z1.h
; CHECK-NEXT:    stp q2, q0, [x1, #32]
; CHECK-NEXT:    stp q3, q1, [x1]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v16i16_v16i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #160
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    stp q1, q0, [sp]
; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #4]
; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #6]
; NONEON-NOSVE-NEXT:    ldrh w3, [sp, #2]
; NONEON-NOSVE-NEXT:    ldrh w5, [sp]
; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #12]
; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #14]
; NONEON-NOSVE-NEXT:    add w13, w13, w13
; NONEON-NOSVE-NEXT:    add w14, w14, w14
; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #10]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #46]
; NONEON-NOSVE-NEXT:    add w14, w3, w3
; NONEON-NOSVE-NEXT:    strh w13, [sp, #44]
; NONEON-NOSVE-NEXT:    add w13, w5, w5
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #42]
; NONEON-NOSVE-NEXT:    add w14, w4, w4
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w13, [sp, #40]
; NONEON-NOSVE-NEXT:    add w13, w2, w2
; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #22]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #38]
; NONEON-NOSVE-NEXT:    add w14, w0, w0
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    strh w13, [sp, #36]
; NONEON-NOSVE-NEXT:    add w13, w18, w18
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    strh w14, [sp, #34]
; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #28]
; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #30]
; NONEON-NOSVE-NEXT:    strh w13, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #18]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #20]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #50]
; NONEON-NOSVE-NEXT:    add w14, w17, w17
; NONEON-NOSVE-NEXT:    add w12, w12, w12
; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
; NONEON-NOSVE-NEXT:    add w13, w16, w16
; NONEON-NOSVE-NEXT:    add w11, w11, w11
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    add w10, w10, w10
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #78]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #76]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #62]
; NONEON-NOSVE-NEXT:    add w14, w15, w15
; NONEON-NOSVE-NEXT:    strh w13, [sp, #60]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #74]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #72]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #58]
; NONEON-NOSVE-NEXT:    strh w12, [sp, #56]
; NONEON-NOSVE-NEXT:    strh w11, [sp, #54]
; NONEON-NOSVE-NEXT:    strh w10, [sp, #52]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #70]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #68]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #66]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #64]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #94]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #92]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #96]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #90]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #88]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #86]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #84]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #82]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #80]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
; NONEON-NOSVE-NEXT:    add sp, sp, #160
; NONEON-NOSVE-NEXT:    ret
  %a = load <16 x i16>, ptr %in
  %b = add <16 x i16> %a, %a
  %c = sext <16 x i16> %b to <16 x i32>
  store <16 x i32> %c, ptr %out
  ret void
}

;
; sext i16 -> i64
;

define void @sext_v4i16_v4i64(<4 x i16> %a, ptr %out) {
; CHECK-LABEL: sext_v4i16_v4i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
; CHECK-NEXT:    sunpklo z0.s, z0.h
; CHECK-NEXT:    sunpklo z1.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z0.d, z0.s
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v4i16_v4i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #80
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #10]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #8]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #14]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #12]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #40]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #64]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #32]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #48]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #80
; NONEON-NOSVE-NEXT:    ret
  %b = sext <4 x i16> %a to <4 x i64>
  store <4 x i64>%b, ptr %out
  ret void
}

define void @sext_v8i16_v8i64(<8 x i16> %a, ptr %out) {
; CHECK-LABEL: sext_v8i16_v8i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    sunpklo z1.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z0.s, z0.h
; CHECK-NEXT:    sunpklo z2.d, z1.s
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    sunpklo z1.d, z1.s
; CHECK-NEXT:    sunpklo z3.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z0.d, z0.s
; CHECK-NEXT:    stp q2, q1, [x0]
; CHECK-NEXT:    stp q3, q0, [x0, #32]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v8i16_v8i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    str q0, [sp, #-160]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #28]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #22]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #20]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #88]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #144]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #80]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #128]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #72]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #128]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #112]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #64]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
; NONEON-NOSVE-NEXT:    add sp, sp, #160
; NONEON-NOSVE-NEXT:    ret
  %b = sext <8 x i16> %a to <8 x i64>
  store <8 x i64>%b, ptr %out
  ret void
}

define void @sext_v16i16_v16i64(ptr %in, ptr %out) {
; CHECK-LABEL: sext_v16i16_v16i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldp q1, q0, [x0]
; CHECK-NEXT:    add z0.h, z0.h, z0.h
; CHECK-NEXT:    add z1.h, z1.h, z1.h
; CHECK-NEXT:    sunpklo z2.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z3.s, z1.h
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    sunpklo z0.s, z0.h
; CHECK-NEXT:    sunpklo z1.s, z1.h
; CHECK-NEXT:    sunpklo z4.d, z2.s
; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
; CHECK-NEXT:    sunpklo z5.d, z3.s
; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
; CHECK-NEXT:    sunpklo z6.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z2.d, z2.s
; CHECK-NEXT:    sunpklo z7.d, z1.s
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    sunpklo z3.d, z3.s
; CHECK-NEXT:    sunpklo z0.d, z0.s
; CHECK-NEXT:    sunpklo z1.d, z1.s
; CHECK-NEXT:    stp q4, q2, [x1, #64]
; CHECK-NEXT:    stp q5, q3, [x1]
; CHECK-NEXT:    stp q6, q0, [x1, #96]
; CHECK-NEXT:    stp q7, q1, [x1, #32]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v16i16_v16i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #368
; NONEON-NOSVE-NEXT:    str x29, [sp, #352] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 368
; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    ldr x29, [sp, #352] // 8-byte Folded Reload
; NONEON-NOSVE-NEXT:    stp q1, q0, [sp]
; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #4]
; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #6]
; NONEON-NOSVE-NEXT:    ldrh w3, [sp, #2]
; NONEON-NOSVE-NEXT:    ldrh w5, [sp]
; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #12]
; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #14]
; NONEON-NOSVE-NEXT:    add w13, w13, w13
; NONEON-NOSVE-NEXT:    add w14, w14, w14
; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #10]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #54]
; NONEON-NOSVE-NEXT:    add w14, w3, w3
; NONEON-NOSVE-NEXT:    strh w13, [sp, #52]
; NONEON-NOSVE-NEXT:    add w13, w5, w5
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #50]
; NONEON-NOSVE-NEXT:    add w14, w4, w4
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w13, [sp, #48]
; NONEON-NOSVE-NEXT:    add w13, w2, w2
; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #22]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #46]
; NONEON-NOSVE-NEXT:    add w14, w0, w0
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    strh w13, [sp, #44]
; NONEON-NOSVE-NEXT:    add w13, w18, w18
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    strh w14, [sp, #42]
; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #28]
; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #30]
; NONEON-NOSVE-NEXT:    strh w13, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #18]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #20]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #58]
; NONEON-NOSVE-NEXT:    add w14, w17, w17
; NONEON-NOSVE-NEXT:    add w12, w12, w12
; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
; NONEON-NOSVE-NEXT:    add w13, w16, w16
; NONEON-NOSVE-NEXT:    add w11, w11, w11
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #72]
; NONEON-NOSVE-NEXT:    add w10, w10, w10
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #82]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #80]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #70]
; NONEON-NOSVE-NEXT:    add w14, w15, w15
; NONEON-NOSVE-NEXT:    strh w13, [sp, #68]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #86]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #84]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #66]
; NONEON-NOSVE-NEXT:    strh w12, [sp, #64]
; NONEON-NOSVE-NEXT:    strh w11, [sp, #62]
; NONEON-NOSVE-NEXT:    strh w10, [sp, #60]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #56]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #74]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #72]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #78]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #76]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #88]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #120]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #98]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #96]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #102]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #100]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #184]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #104]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #160]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #90]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #88]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #94]
; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #92]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #168]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #152]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
; NONEON-NOSVE-NEXT:    str d0, [sp, #360]
; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp, #136]
; NONEON-NOSVE-NEXT:    str d2, [sp, #200]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #184]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #256]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #192]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #272]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #168]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #256]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #224]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #176]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #240]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #216]
; NONEON-NOSVE-NEXT:    ldp q3, q4, [sp, #224]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #320]
; NONEON-NOSVE-NEXT:    ldrsw x9, [sp, #364]
; NONEON-NOSVE-NEXT:    ldrsw x8, [sp, #360]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #336]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #200]
; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #320]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #288]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #208]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #304]
; NONEON-NOSVE-NEXT:    ldp q5, q2, [sp, #288]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
; NONEON-NOSVE-NEXT:    stp q3, q4, [x1, #32]
; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #64]
; NONEON-NOSVE-NEXT:    stp q5, q2, [x1, #96]
; NONEON-NOSVE-NEXT:    add sp, sp, #368
; NONEON-NOSVE-NEXT:    ret
  %a = load <16 x i16>, ptr %in
  %b = add <16 x i16> %a, %a
  %c = sext <16 x i16> %b to <16 x i64>
  store <16 x i64> %c, ptr %out
  ret void
}

;
; sext i32 -> i64
;

define void @sext_v4i32_v4i64(<4 x i32> %a, ptr %out) {
; CHECK-LABEL: sext_v4i32_v4i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    sunpklo z1.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z0.d, z0.s
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v4i32_v4i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    str q0, [sp, #-64]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #24]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #48]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #16]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #64
; NONEON-NOSVE-NEXT:    ret
  %b = sext <4 x i32> %a to <4 x i64>
  store <4 x i64>%b, ptr %out
  ret void
}

define void @sext_v8i32_v8i64(ptr %in, ptr %out) {
; CHECK-LABEL: sext_v8i32_v8i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldp q1, q0, [x0]
; CHECK-NEXT:    add z0.s, z0.s, z0.s
; CHECK-NEXT:    add z1.s, z1.s, z1.s
; CHECK-NEXT:    sunpklo z2.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    sunpklo z3.d, z1.s
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    sunpklo z0.d, z0.s
; CHECK-NEXT:    sunpklo z1.d, z1.s
; CHECK-NEXT:    stp q2, q0, [x1, #32]
; CHECK-NEXT:    stp q3, q1, [x1]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: sext_v8i32_v8i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #160
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    stp q1, q0, [sp]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp]
; NONEON-NOSVE-NEXT:    ldp w12, w13, [sp, #8]
; NONEON-NOSVE-NEXT:    ldp w14, w15, [sp, #16]
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    ldp w10, w11, [sp, #24]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
; NONEON-NOSVE-NEXT:    add w9, w13, w13
; NONEON-NOSVE-NEXT:    add w8, w12, w12
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
; NONEON-NOSVE-NEXT:    add w9, w15, w15
; NONEON-NOSVE-NEXT:    add w8, w14, w14
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
; NONEON-NOSVE-NEXT:    add w9, w11, w11
; NONEON-NOSVE-NEXT:    add w8, w10, w10
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #72]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #112]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #64]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #96]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #88]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #96]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #144]
; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #80]
; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #128]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
; NONEON-NOSVE-NEXT:    add sp, sp, #160
; NONEON-NOSVE-NEXT:    ret
  %a = load <8 x i32>, ptr %in
  %b = add <8 x i32> %a, %a
  %c = sext <8 x i32> %b to <8 x i64>
  store <8 x i64> %c, ptr %out
  ret void
}

;
; zext i8 -> i16
;

define void @zext_v16i8_v16i16(<16 x i8> %a, ptr %out) {
; CHECK-LABEL: zext_v16i8_v16i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    uunpklo z1.h, z0.b
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.h, z0.b
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v16i8_v16i16:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    str q0, [sp, #-64]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #64
; NONEON-NOSVE-NEXT:    ret
  %b = zext <16 x i8> %a to <16 x i16>
  store <16 x i16>%b, ptr %out
  ret void
}

; NOTE: Extra 'add' is to prevent the extend being combined with the load.
define void @zext_v32i8_v32i16(ptr %in, ptr %out) {
; CHECK-LABEL: zext_v32i8_v32i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldp q1, q0, [x0]
; CHECK-NEXT:    add z0.b, z0.b, z0.b
; CHECK-NEXT:    add z1.b, z1.b, z1.b
; CHECK-NEXT:    uunpklo z2.h, z0.b
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z3.h, z1.b
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    uunpklo z0.h, z0.b
; CHECK-NEXT:    uunpklo z1.h, z1.b
; CHECK-NEXT:    stp q2, q0, [x1, #32]
; CHECK-NEXT:    stp q3, q1, [x1]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v32i8_v32i16:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #272
; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #176] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #192] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #208] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #224] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #240] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #256] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 272
; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88
; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #41]
; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #28]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #38]
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #17]
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #22]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
; NONEON-NOSVE-NEXT:    add w8, w29, w29
; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #36]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
; NONEON-NOSVE-NEXT:    add w8, w27, w27
; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #31]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
; NONEON-NOSVE-NEXT:    add w8, w25, w25
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
; NONEON-NOSVE-NEXT:    add w8, w23, w23
; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #34]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
; NONEON-NOSVE-NEXT:    add w8, w21, w21
; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #29]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
; NONEON-NOSVE-NEXT:    add w8, w19, w19
; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #23]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]
; NONEON-NOSVE-NEXT:    add w9, w28, w28
; NONEON-NOSVE-NEXT:    add w18, w16, w16
; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
; NONEON-NOSVE-NEXT:    add w8, w6, w6
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #27]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]
; NONEON-NOSVE-NEXT:    add w9, w26, w26
; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
; NONEON-NOSVE-NEXT:    add w8, w4, w4
; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #25]
; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]
; NONEON-NOSVE-NEXT:    add w9, w24, w24
; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
; NONEON-NOSVE-NEXT:    add w8, w2, w2
; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
; NONEON-NOSVE-NEXT:    add w17, w17, w17
; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]
; NONEON-NOSVE-NEXT:    add w9, w22, w22
; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
; NONEON-NOSVE-NEXT:    add w8, w16, w16
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
; NONEON-NOSVE-NEXT:    strb w17, [sp, #63]
; NONEON-NOSVE-NEXT:    add w17, w30, w30
; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]
; NONEON-NOSVE-NEXT:    add w9, w20, w20
; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #39]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
; NONEON-NOSVE-NEXT:    add w8, w14, w14
; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #37]
; NONEON-NOSVE-NEXT:    strb w18, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #35]
; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #33]
; NONEON-NOSVE-NEXT:    strb w17, [sp, #61]
; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #47]
; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #45]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]
; NONEON-NOSVE-NEXT:    add w9, w7, w7
; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #43]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
; NONEON-NOSVE-NEXT:    add w8, w12, w12
; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
; NONEON-NOSVE-NEXT:    add w8, w10, w10
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #79]
; NONEON-NOSVE-NEXT:    add w9, w5, w5
; NONEON-NOSVE-NEXT:    strb w9, [sp, #77]
; NONEON-NOSVE-NEXT:    add w9, w3, w3
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #95]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #75]
; NONEON-NOSVE-NEXT:    add w9, w0, w0
; NONEON-NOSVE-NEXT:    strh w8, [sp, #142]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #94]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #73]
; NONEON-NOSVE-NEXT:    add w9, w15, w15
; NONEON-NOSVE-NEXT:    strh w8, [sp, #140]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #93]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #71]
; NONEON-NOSVE-NEXT:    add w9, w13, w13
; NONEON-NOSVE-NEXT:    strh w8, [sp, #138]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #92]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #69]
; NONEON-NOSVE-NEXT:    add w9, w11, w11
; NONEON-NOSVE-NEXT:    strh w8, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #91]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #67]
; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #134]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #90]
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #256] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #132]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #89]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #240] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #130]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #88]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #87]
; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #224] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #126]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #86]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #208] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #124]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #85]
; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #192] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #122]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #84]
; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #176] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #83]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #118]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #82]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #116]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #81]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #114]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #80]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #111]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #112]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #174]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #110]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #172]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #109]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #170]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #108]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #168]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #107]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #166]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #106]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #164]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #105]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #162]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #104]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #160]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #103]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #158]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #102]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #156]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #101]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #154]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #100]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #99]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #150]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #98]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #148]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #97]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #146]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #96]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #144]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #144]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
; NONEON-NOSVE-NEXT:    add sp, sp, #272
; NONEON-NOSVE-NEXT:    ret
  %a = load <32 x i8>, ptr %in
  %b = add <32 x i8> %a, %a
  %c = zext <32 x i8> %b to <32 x i16>
  store <32 x i16> %c, ptr %out
  ret void
}

;
; zext i8 -> i32
;

define void @zext_v8i8_v8i32(<8 x i8> %a, ptr %out) {
; CHECK-LABEL: zext_v8i8_v8i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
; CHECK-NEXT:    uunpklo z0.h, z0.b
; CHECK-NEXT:    uunpklo z1.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v8i8_v8i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #80
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #11]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #9]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #15]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #22]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #13]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #44]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #42]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #40]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #38]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #36]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #34]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #32]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #80
; NONEON-NOSVE-NEXT:    ret
  %b = zext <8 x i8> %a to <8 x i32>
  store <8 x i32>%b, ptr %out
  ret void
}

define void @zext_v16i8_v16i32(<16 x i8> %a, ptr %out) {
; CHECK-LABEL: zext_v16i8_v16i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    uunpklo z1.h, z0.b
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.h, z0.b
; CHECK-NEXT:    uunpklo z2.s, z1.h
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    uunpklo z1.s, z1.h
; CHECK-NEXT:    uunpklo z3.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    stp q2, q1, [x0]
; CHECK-NEXT:    stp q3, q0, [x0, #32]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v16i8_v16i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    str q0, [sp, #-160]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #94]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #92]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #90]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #88]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #86]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #84]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #82]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #80]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #78]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #76]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #128]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #74]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #72]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #70]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #68]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #66]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #64]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
; NONEON-NOSVE-NEXT:    add sp, sp, #160
; NONEON-NOSVE-NEXT:    ret
  %b = zext <16 x i8> %a to <16 x i32>
  store <16 x i32> %b, ptr %out
  ret void
}

define void @zext_v32i8_v32i32(ptr %in, ptr %out) {
; CHECK-LABEL: zext_v32i8_v32i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldp q1, q0, [x0]
; CHECK-NEXT:    add z0.b, z0.b, z0.b
; CHECK-NEXT:    add z1.b, z1.b, z1.b
; CHECK-NEXT:    uunpklo z2.h, z0.b
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z3.h, z1.b
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    uunpklo z0.h, z0.b
; CHECK-NEXT:    uunpklo z1.h, z1.b
; CHECK-NEXT:    uunpklo z4.s, z2.h
; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
; CHECK-NEXT:    uunpklo z5.s, z3.h
; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
; CHECK-NEXT:    uunpklo z6.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z2.s, z2.h
; CHECK-NEXT:    uunpklo z7.s, z1.h
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    uunpklo z3.s, z3.h
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    uunpklo z1.s, z1.h
; CHECK-NEXT:    stp q4, q2, [x1, #64]
; CHECK-NEXT:    stp q5, q3, [x1]
; CHECK-NEXT:    stp q6, q0, [x1, #96]
; CHECK-NEXT:    stp q7, q1, [x1, #32]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v32i8_v32i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #464
; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #368] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #384] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #400] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #416] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #432] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #448] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 464
; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88
; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #41]
; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #28]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #38]
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #17]
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #22]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
; NONEON-NOSVE-NEXT:    add w8, w29, w29
; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #36]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
; NONEON-NOSVE-NEXT:    add w8, w27, w27
; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #31]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
; NONEON-NOSVE-NEXT:    add w8, w25, w25
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
; NONEON-NOSVE-NEXT:    add w8, w23, w23
; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #34]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
; NONEON-NOSVE-NEXT:    add w8, w21, w21
; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #29]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
; NONEON-NOSVE-NEXT:    add w8, w19, w19
; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #23]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]
; NONEON-NOSVE-NEXT:    add w9, w28, w28
; NONEON-NOSVE-NEXT:    add w18, w16, w16
; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
; NONEON-NOSVE-NEXT:    add w8, w6, w6
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #27]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]
; NONEON-NOSVE-NEXT:    add w9, w26, w26
; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
; NONEON-NOSVE-NEXT:    add w8, w4, w4
; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #25]
; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]
; NONEON-NOSVE-NEXT:    add w9, w24, w24
; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
; NONEON-NOSVE-NEXT:    add w8, w2, w2
; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
; NONEON-NOSVE-NEXT:    add w17, w17, w17
; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]
; NONEON-NOSVE-NEXT:    add w9, w22, w22
; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
; NONEON-NOSVE-NEXT:    add w8, w16, w16
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
; NONEON-NOSVE-NEXT:    strb w17, [sp, #63]
; NONEON-NOSVE-NEXT:    add w17, w30, w30
; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]
; NONEON-NOSVE-NEXT:    add w9, w20, w20
; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #39]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
; NONEON-NOSVE-NEXT:    add w8, w14, w14
; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #37]
; NONEON-NOSVE-NEXT:    strb w18, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #35]
; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #33]
; NONEON-NOSVE-NEXT:    strb w17, [sp, #61]
; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #47]
; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #45]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]
; NONEON-NOSVE-NEXT:    add w9, w7, w7
; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #43]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
; NONEON-NOSVE-NEXT:    add w8, w12, w12
; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
; NONEON-NOSVE-NEXT:    add w8, w10, w10
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #79]
; NONEON-NOSVE-NEXT:    add w9, w5, w5
; NONEON-NOSVE-NEXT:    strb w9, [sp, #77]
; NONEON-NOSVE-NEXT:    add w9, w3, w3
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #91]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #75]
; NONEON-NOSVE-NEXT:    add w9, w0, w0
; NONEON-NOSVE-NEXT:    strh w8, [sp, #134]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #90]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #73]
; NONEON-NOSVE-NEXT:    add w9, w15, w15
; NONEON-NOSVE-NEXT:    strh w8, [sp, #132]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #89]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #71]
; NONEON-NOSVE-NEXT:    add w9, w13, w13
; NONEON-NOSVE-NEXT:    strh w8, [sp, #130]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #88]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #69]
; NONEON-NOSVE-NEXT:    add w9, w11, w11
; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #95]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #67]
; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #142]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #94]
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #448] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #140]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #93]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #432] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #138]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #92]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #83]
; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #416] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #118]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #82]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #128]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #116]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #81]
; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #400] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #114]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #80]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #192]
; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #384] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #87]
; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #368] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #126]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #86]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #124]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #85]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #122]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #84]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #107]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #112]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #166]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #106]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #164]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #105]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #176]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #162]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #182]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #160]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #111]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #174]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #110]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #172]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #109]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #170]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #108]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #168]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #99]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #160]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #150]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #98]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #148]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #97]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #224]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #146]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #96]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #144]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #103]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #158]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #102]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #156]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #101]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #154]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #100]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #198]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #144]
; NONEON-NOSVE-NEXT:    str w8, [sp, #284]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #196]
; NONEON-NOSVE-NEXT:    str w8, [sp, #280]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #194]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #208]
; NONEON-NOSVE-NEXT:    str w8, [sp, #276]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #192]
; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #206]
; NONEON-NOSVE-NEXT:    str w8, [sp, #300]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #204]
; NONEON-NOSVE-NEXT:    str w8, [sp, #296]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #202]
; NONEON-NOSVE-NEXT:    str w8, [sp, #292]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #200]
; NONEON-NOSVE-NEXT:    str w8, [sp, #288]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #180]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #272]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #248]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #178]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #176]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #240]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #190]
; NONEON-NOSVE-NEXT:    str w8, [sp, #268]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #188]
; NONEON-NOSVE-NEXT:    str w8, [sp, #264]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #186]
; NONEON-NOSVE-NEXT:    str w8, [sp, #260]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #184]
; NONEON-NOSVE-NEXT:    str w8, [sp, #256]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #230]
; NONEON-NOSVE-NEXT:    ldp q3, q4, [sp, #240]
; NONEON-NOSVE-NEXT:    str w8, [sp, #348]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #228]
; NONEON-NOSVE-NEXT:    str w8, [sp, #344]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #226]
; NONEON-NOSVE-NEXT:    str w8, [sp, #340]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #224]
; NONEON-NOSVE-NEXT:    str w8, [sp, #336]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #238]
; NONEON-NOSVE-NEXT:    str w8, [sp, #364]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #236]
; NONEON-NOSVE-NEXT:    str w8, [sp, #360]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #234]
; NONEON-NOSVE-NEXT:    str w8, [sp, #356]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #232]
; NONEON-NOSVE-NEXT:    str w8, [sp, #352]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #214]
; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #336]
; NONEON-NOSVE-NEXT:    str w8, [sp, #316]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #212]
; NONEON-NOSVE-NEXT:    str w8, [sp, #312]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #210]
; NONEON-NOSVE-NEXT:    str w8, [sp, #308]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #208]
; NONEON-NOSVE-NEXT:    str w8, [sp, #304]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #222]
; NONEON-NOSVE-NEXT:    str w8, [sp, #332]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #220]
; NONEON-NOSVE-NEXT:    str w8, [sp, #328]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #218]
; NONEON-NOSVE-NEXT:    str w8, [sp, #324]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #216]
; NONEON-NOSVE-NEXT:    str w8, [sp, #320]
; NONEON-NOSVE-NEXT:    ldp q5, q2, [sp, #304]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
; NONEON-NOSVE-NEXT:    stp q3, q4, [x1, #32]
; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #64]
; NONEON-NOSVE-NEXT:    stp q5, q2, [x1, #96]
; NONEON-NOSVE-NEXT:    add sp, sp, #464
; NONEON-NOSVE-NEXT:    ret
  %a = load <32 x i8>, ptr %in
  %b = add <32 x i8> %a, %a
  %c = zext <32 x i8> %b to <32 x i32>
  store <32 x i32> %c, ptr %out
  ret void
}

;
; zext i8 -> i64
;

; NOTE: v4i8 is an unpacked typed stored within a v4i16 container. The zero
; extend is a two step process where the container is zero_extend_inreg'd with
; the result feeding a normal zero extend from halfs to doublewords.
define void @zext_v4i8_v4i64(<4 x i8> %a, ptr %out) {
; CHECK-LABEL: zext_v4i8_v4i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
; CHECK-NEXT:    and z0.h, z0.h, #0xff
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    uunpklo z1.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.d, z0.s
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v4i8_v4i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #80
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #40]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #72]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #64]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #32]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #56]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #48]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #80
; NONEON-NOSVE-NEXT:    ret
  %b = zext <4 x i8> %a to <4 x i64>
  store <4 x i64>%b, ptr %out
  ret void
}

define void @zext_v8i8_v8i64(<8 x i8> %a, ptr %out) {
; CHECK-LABEL: zext_v8i8_v8i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
; CHECK-NEXT:    uunpklo z0.h, z0.b
; CHECK-NEXT:    uunpklo z1.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    uunpklo z2.d, z1.s
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    uunpklo z3.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z1.d, z1.s
; CHECK-NEXT:    uunpklo z0.d, z0.s
; CHECK-NEXT:    stp q2, q1, [x0]
; CHECK-NEXT:    stp q3, q0, [x0, #32]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v8i8_v8i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #176
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 176
; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
; NONEON-NOSVE-NEXT:    add x8, sp, #144
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #28]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #22]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #64]
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #44]
; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #72]
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #34]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #64]
; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #38]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #36]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #96]
; NONEON-NOSVE-NEXT:    stp w9, w10, [sp, #56]
; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #48]
; NONEON-NOSVE-NEXT:    stp w10, wzr, [sp, #152]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #144]
; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #104]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #80]
; NONEON-NOSVE-NEXT:    stp w10, wzr, [sp, #168]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #160]
; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #80]
; NONEON-NOSVE-NEXT:    stp w10, wzr, [sp, #120]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #112]
; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #88]
; NONEON-NOSVE-NEXT:    stp w10, wzr, [sp, #136]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #128]
; NONEON-NOSVE-NEXT:    ldp q1, q0, [sp, #112]
; NONEON-NOSVE-NEXT:    ldp q2, q3, [x8]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #176
; NONEON-NOSVE-NEXT:    ret
  %b = zext <8 x i8> %a to <8 x i64>
  store <8 x i64>%b, ptr %out
  ret void
}

define void @zext_v16i8_v16i64(<16 x i8> %a, ptr %out) {
; CHECK-LABEL: zext_v16i8_v16i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    uunpklo z1.h, z0.b
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.h, z0.b
; CHECK-NEXT:    uunpklo z2.s, z1.h
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    uunpklo z1.s, z1.h
; CHECK-NEXT:    uunpklo z3.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z4.d, z2.s
; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    uunpklo z6.d, z1.s
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    uunpklo z5.d, z3.s
; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
; CHECK-NEXT:    uunpklo z2.d, z2.s
; CHECK-NEXT:    uunpklo z1.d, z1.s
; CHECK-NEXT:    uunpklo z7.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z3.d, z3.s
; CHECK-NEXT:    stp q4, q2, [x0]
; CHECK-NEXT:    uunpklo z0.d, z0.s
; CHECK-NEXT:    stp q6, q1, [x0, #32]
; CHECK-NEXT:    stp q5, q3, [x0, #64]
; CHECK-NEXT:    stp q7, q0, [x0, #96]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v16i8_v16i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #368
; NONEON-NOSVE-NEXT:    str x29, [sp, #352] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 368
; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
; NONEON-NOSVE-NEXT:    str q0, [sp]
; NONEON-NOSVE-NEXT:    ldr x29, [sp, #352] // 8-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #332]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #324]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #348]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #35]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #340]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #300]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #70]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #34]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #292]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #68]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #33]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #316]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #66]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #32]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #308]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #64]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #39]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #268]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #38]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #260]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #37]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #284]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #36]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #276]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #56]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #88]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #98]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #40]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #102]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #100]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #160]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #90]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #88]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #72]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #152]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #94]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #92]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #82]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #80]
; NONEON-NOSVE-NEXT:    str d0, [sp, #360]
; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp, #136]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #86]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #84]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #74]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #72]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #120]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #78]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #76]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #216]
; NONEON-NOSVE-NEXT:    stp d0, d2, [sp, #192]
; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp, #104]
; NONEON-NOSVE-NEXT:    str w8, [sp, #320]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #364]
; NONEON-NOSVE-NEXT:    str w9, [sp, #328]
; NONEON-NOSVE-NEXT:    str w8, [sp, #344]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #360]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #176]
; NONEON-NOSVE-NEXT:    str w8, [sp, #336]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #200]
; NONEON-NOSVE-NEXT:    str d2, [sp, #168]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #320]
; NONEON-NOSVE-NEXT:    str w9, [sp, #296]
; NONEON-NOSVE-NEXT:    str w8, [sp, #288]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #208]
; NONEON-NOSVE-NEXT:    str w9, [sp, #312]
; NONEON-NOSVE-NEXT:    str w8, [sp, #304]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #184]
; NONEON-NOSVE-NEXT:    ldp q3, q4, [sp, #288]
; NONEON-NOSVE-NEXT:    str w9, [sp, #264]
; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #252]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #192]
; NONEON-NOSVE-NEXT:    str w9, [sp, #280]
; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #168]
; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #256]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #232]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #224]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #176]
; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #244]
; NONEON-NOSVE-NEXT:    str w8, [sp, #240]
; NONEON-NOSVE-NEXT:    ldp q5, q2, [sp, #224]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
; NONEON-NOSVE-NEXT:    stp q3, q4, [x0, #32]
; NONEON-NOSVE-NEXT:    stp q6, q7, [x0, #64]
; NONEON-NOSVE-NEXT:    stp q5, q2, [x0, #96]
; NONEON-NOSVE-NEXT:    add sp, sp, #368
; NONEON-NOSVE-NEXT:    ret
  %b = zext <16 x i8> %a to <16 x i64>
  store <16 x i64> %b, ptr %out
  ret void
}

define void @zext_v32i8_v32i64(ptr %in, ptr %out) {
; CHECK-LABEL: zext_v32i8_v32i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldp q1, q0, [x0]
; CHECK-NEXT:    add z0.b, z0.b, z0.b
; CHECK-NEXT:    add z1.b, z1.b, z1.b
; CHECK-NEXT:    mov z2.d, z0.d
; CHECK-NEXT:    uunpklo z0.h, z0.b
; CHECK-NEXT:    mov z3.d, z1.d
; CHECK-NEXT:    uunpklo z1.h, z1.b
; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
; CHECK-NEXT:    uunpklo z4.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z5.s, z1.h
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    uunpklo z2.h, z2.b
; CHECK-NEXT:    uunpklo z3.h, z3.b
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    uunpklo z16.d, z4.s
; CHECK-NEXT:    ext z4.b, z4.b, z4.b, #8
; CHECK-NEXT:    uunpklo z1.s, z1.h
; CHECK-NEXT:    uunpklo z17.d, z5.s
; CHECK-NEXT:    ext z5.b, z5.b, z5.b, #8
; CHECK-NEXT:    uunpklo z6.s, z2.h
; CHECK-NEXT:    uunpklo z7.s, z3.h
; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
; CHECK-NEXT:    uunpklo z4.d, z4.s
; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
; CHECK-NEXT:    uunpklo z19.d, z0.s
; CHECK-NEXT:    uunpklo z5.d, z5.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z2.s, z2.h
; CHECK-NEXT:    uunpklo z18.d, z6.s
; CHECK-NEXT:    ext z6.b, z6.b, z6.b, #8
; CHECK-NEXT:    uunpklo z3.s, z3.h
; CHECK-NEXT:    stp q16, q4, [x1, #128]
; CHECK-NEXT:    mov z16.d, z7.d
; CHECK-NEXT:    uunpklo z0.d, z0.s
; CHECK-NEXT:    stp q17, q5, [x1]
; CHECK-NEXT:    uunpklo z5.d, z7.s
; CHECK-NEXT:    uunpklo z4.d, z6.s
; CHECK-NEXT:    mov z6.d, z1.d
; CHECK-NEXT:    ext z16.b, z16.b, z7.b, #8
; CHECK-NEXT:    mov z7.d, z2.d
; CHECK-NEXT:    stp q19, q0, [x1, #160]
; CHECK-NEXT:    uunpklo z0.d, z2.s
; CHECK-NEXT:    ext z6.b, z6.b, z1.b, #8
; CHECK-NEXT:    uunpklo z1.d, z1.s
; CHECK-NEXT:    stp q18, q4, [x1, #192]
; CHECK-NEXT:    mov z4.d, z3.d
; CHECK-NEXT:    ext z7.b, z7.b, z2.b, #8
; CHECK-NEXT:    uunpklo z16.d, z16.s
; CHECK-NEXT:    uunpklo z6.d, z6.s
; CHECK-NEXT:    ext z4.b, z4.b, z3.b, #8
; CHECK-NEXT:    uunpklo z2.d, z7.s
; CHECK-NEXT:    uunpklo z3.d, z3.s
; CHECK-NEXT:    stp q5, q16, [x1, #64]
; CHECK-NEXT:    stp q1, q6, [x1, #32]
; CHECK-NEXT:    uunpklo z1.d, z4.s
; CHECK-NEXT:    stp q0, q2, [x1, #224]
; CHECK-NEXT:    stp q3, q1, [x1, #96]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v32i8_v32i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #-96]! // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #16] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #32] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #48] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #64] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
; NONEON-NOSVE-NEXT:    sub sp, sp, #752
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 848
; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8
; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16
; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24
; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32
; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40
; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48
; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56
; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64
; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72
; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80
; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88
; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #572]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #564]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #588]
; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #41]
; NONEON-NOSVE-NEXT:    ldrb w29, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #28]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #38]
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    ldrb w28, [sp, #17]
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #22]
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
; NONEON-NOSVE-NEXT:    add w8, w29, w29
; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
; NONEON-NOSVE-NEXT:    add w8, w27, w27
; NONEON-NOSVE-NEXT:    add w18, w16, w16
; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
; NONEON-NOSVE-NEXT:    add w8, w25, w25
; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #36]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
; NONEON-NOSVE-NEXT:    add w8, w23, w23
; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #31]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
; NONEON-NOSVE-NEXT:    add w8, w21, w21
; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #34]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
; NONEON-NOSVE-NEXT:    add w8, w19, w19
; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #29]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]
; NONEON-NOSVE-NEXT:    add w9, w28, w28
; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #23]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
; NONEON-NOSVE-NEXT:    add w8, w6, w6
; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #27]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]
; NONEON-NOSVE-NEXT:    add w9, w26, w26
; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
; NONEON-NOSVE-NEXT:    add w8, w4, w4
; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #46]
; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #25]
; NONEON-NOSVE-NEXT:    ldrb w30, [sp, #21]
; NONEON-NOSVE-NEXT:    add w17, w17, w17
; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]
; NONEON-NOSVE-NEXT:    add w9, w24, w24
; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #44]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
; NONEON-NOSVE-NEXT:    add w8, w2, w2
; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #42]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]
; NONEON-NOSVE-NEXT:    add w9, w22, w22
; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #39]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
; NONEON-NOSVE-NEXT:    add w8, w16, w16
; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #37]
; NONEON-NOSVE-NEXT:    strb w17, [sp, #63]
; NONEON-NOSVE-NEXT:    add w17, w30, w30
; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #35]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]
; NONEON-NOSVE-NEXT:    add w9, w20, w20
; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #33]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
; NONEON-NOSVE-NEXT:    add w8, w14, w14
; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #47]
; NONEON-NOSVE-NEXT:    strb w18, [sp, #62]
; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #45]
; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #43]
; NONEON-NOSVE-NEXT:    strb w17, [sp, #61]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]
; NONEON-NOSVE-NEXT:    add w9, w7, w7
; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
; NONEON-NOSVE-NEXT:    add w8, w12, w12
; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
; NONEON-NOSVE-NEXT:    add w8, w10, w10
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
; NONEON-NOSVE-NEXT:    strb w9, [sp, #79]
; NONEON-NOSVE-NEXT:    add w9, w5, w5
; NONEON-NOSVE-NEXT:    strb w9, [sp, #77]
; NONEON-NOSVE-NEXT:    add w9, w3, w3
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #91]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #75]
; NONEON-NOSVE-NEXT:    add w9, w0, w0
; NONEON-NOSVE-NEXT:    strh w8, [sp, #134]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #90]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #73]
; NONEON-NOSVE-NEXT:    add w9, w15, w15
; NONEON-NOSVE-NEXT:    strh w8, [sp, #132]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #89]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #71]
; NONEON-NOSVE-NEXT:    add w9, w13, w13
; NONEON-NOSVE-NEXT:    strh w8, [sp, #130]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #88]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #69]
; NONEON-NOSVE-NEXT:    add w9, w11, w11
; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #95]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #67]
; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
; NONEON-NOSVE-NEXT:    strh w8, [sp, #142]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #94]
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    str wzr, [sp, #580]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #140]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #93]
; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #138]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #92]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #604]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #83]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #596]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #118]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #82]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #128]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #116]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #81]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #620]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #114]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #80]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #192]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #87]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #612]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #126]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #86]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #508]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #124]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #85]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #500]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #122]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #84]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #524]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #107]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #112]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #516]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #166]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #106]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #540]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #164]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #105]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #176]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #162]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #178]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #532]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #160]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #111]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #556]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #174]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #110]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #548]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #172]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #109]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #700]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #170]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #108]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #692]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #168]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #99]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #160]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #716]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #150]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #98]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #708]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #148]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #97]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #224]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #146]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #96]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #732]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #144]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #103]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #724]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #158]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #102]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #748]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #156]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #101]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #740]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #154]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #100]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #636]
; NONEON-NOSVE-NEXT:    strh w8, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #194]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #144]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #628]
; NONEON-NOSVE-NEXT:    str w8, [sp, #276]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #192]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #652]
; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #198]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #208]
; NONEON-NOSVE-NEXT:    str w8, [sp, #284]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #196]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #644]
; NONEON-NOSVE-NEXT:    str w8, [sp, #280]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #202]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #272]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #668]
; NONEON-NOSVE-NEXT:    str w8, [sp, #292]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #200]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #660]
; NONEON-NOSVE-NEXT:    str w8, [sp, #288]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #206]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #400]
; NONEON-NOSVE-NEXT:    str w8, [sp, #300]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #204]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #684]
; NONEON-NOSVE-NEXT:    str w8, [sp, #296]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #176]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #288]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #676]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #240]
; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #182]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #180]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #248]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #186]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #416]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #240]
; NONEON-NOSVE-NEXT:    str w8, [sp, #260]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #184]
; NONEON-NOSVE-NEXT:    str w8, [sp, #256]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #190]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #368]
; NONEON-NOSVE-NEXT:    str w8, [sp, #268]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #188]
; NONEON-NOSVE-NEXT:    str w8, [sp, #264]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #226]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #256]
; NONEON-NOSVE-NEXT:    str w8, [sp, #340]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #224]
; NONEON-NOSVE-NEXT:    str w8, [sp, #336]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #230]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #384]
; NONEON-NOSVE-NEXT:    str w8, [sp, #348]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #228]
; NONEON-NOSVE-NEXT:    str w8, [sp, #344]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #234]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #336]
; NONEON-NOSVE-NEXT:    str w8, [sp, #356]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #232]
; NONEON-NOSVE-NEXT:    str w8, [sp, #352]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #238]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #464]
; NONEON-NOSVE-NEXT:    str w8, [sp, #364]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #236]
; NONEON-NOSVE-NEXT:    str w8, [sp, #360]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #210]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #352]
; NONEON-NOSVE-NEXT:    str w8, [sp, #308]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #208]
; NONEON-NOSVE-NEXT:    str w8, [sp, #304]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #214]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #480]
; NONEON-NOSVE-NEXT:    str w8, [sp, #316]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #212]
; NONEON-NOSVE-NEXT:    str w8, [sp, #312]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #218]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #304]
; NONEON-NOSVE-NEXT:    str w8, [sp, #324]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #216]
; NONEON-NOSVE-NEXT:    str w8, [sp, #320]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #222]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #432]
; NONEON-NOSVE-NEXT:    str w8, [sp, #332]
; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #220]
; NONEON-NOSVE-NEXT:    str w8, [sp, #328]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #404]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #320]
; NONEON-NOSVE-NEXT:    str w8, [sp, #568]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #400]
; NONEON-NOSVE-NEXT:    str w8, [sp, #560]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #412]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #448]
; NONEON-NOSVE-NEXT:    str w8, [sp, #584]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #408]
; NONEON-NOSVE-NEXT:    str w8, [sp, #576]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #420]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #560]
; NONEON-NOSVE-NEXT:    str w8, [sp, #600]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #416]
; NONEON-NOSVE-NEXT:    str w8, [sp, #592]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #428]
; NONEON-NOSVE-NEXT:    str w8, [sp, #616]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #424]
; NONEON-NOSVE-NEXT:    str w8, [sp, #608]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #372]
; NONEON-NOSVE-NEXT:    ldp q2, q3, [sp, #592]
; NONEON-NOSVE-NEXT:    str w8, [sp, #504]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #368]
; NONEON-NOSVE-NEXT:    str w8, [sp, #496]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #380]
; NONEON-NOSVE-NEXT:    str w8, [sp, #520]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #376]
; NONEON-NOSVE-NEXT:    str w8, [sp, #512]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #388]
; NONEON-NOSVE-NEXT:    ldp q4, q5, [sp, #496]
; NONEON-NOSVE-NEXT:    str w8, [sp, #536]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #384]
; NONEON-NOSVE-NEXT:    str w8, [sp, #528]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #396]
; NONEON-NOSVE-NEXT:    str w8, [sp, #552]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #392]
; NONEON-NOSVE-NEXT:    str w8, [sp, #544]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #468]
; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #528]
; NONEON-NOSVE-NEXT:    str w8, [sp, #696]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #464]
; NONEON-NOSVE-NEXT:    str w8, [sp, #688]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #476]
; NONEON-NOSVE-NEXT:    str w8, [sp, #712]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #472]
; NONEON-NOSVE-NEXT:    str w8, [sp, #704]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #484]
; NONEON-NOSVE-NEXT:    ldp q16, q17, [sp, #688]
; NONEON-NOSVE-NEXT:    str w8, [sp, #728]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #480]
; NONEON-NOSVE-NEXT:    str w8, [sp, #720]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #492]
; NONEON-NOSVE-NEXT:    str w8, [sp, #744]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #488]
; NONEON-NOSVE-NEXT:    str w8, [sp, #736]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #436]
; NONEON-NOSVE-NEXT:    ldp q19, q20, [sp, #720]
; NONEON-NOSVE-NEXT:    str w8, [sp, #632]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #432]
; NONEON-NOSVE-NEXT:    str w8, [sp, #624]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #444]
; NONEON-NOSVE-NEXT:    str w8, [sp, #648]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #440]
; NONEON-NOSVE-NEXT:    str w8, [sp, #640]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #452]
; NONEON-NOSVE-NEXT:    ldp q22, q23, [sp, #624]
; NONEON-NOSVE-NEXT:    str w8, [sp, #664]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #448]
; NONEON-NOSVE-NEXT:    str w8, [sp, #656]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #460]
; NONEON-NOSVE-NEXT:    str w8, [sp, #680]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #456]
; NONEON-NOSVE-NEXT:    str w8, [sp, #672]
; NONEON-NOSVE-NEXT:    ldp q21, q18, [sp, #656]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x1, #32]
; NONEON-NOSVE-NEXT:    stp q4, q5, [x1, #64]
; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #96]
; NONEON-NOSVE-NEXT:    stp q16, q17, [x1, #128]
; NONEON-NOSVE-NEXT:    stp q19, q20, [x1, #160]
; NONEON-NOSVE-NEXT:    stp q22, q23, [x1, #192]
; NONEON-NOSVE-NEXT:    stp q21, q18, [x1, #224]
; NONEON-NOSVE-NEXT:    add sp, sp, #752
; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #64] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #48] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #32] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #16] // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp], #96 // 16-byte Folded Reload
; NONEON-NOSVE-NEXT:    ret
  %a = load <32 x i8>, ptr %in
  %b = add <32 x i8> %a, %a
  %c = zext <32 x i8> %b to <32 x i64>
  store <32 x i64> %c, ptr %out
  ret void
}

;
; zext i16 -> i32
;

define void @zext_v8i16_v8i32(<8 x i16> %a, ptr %out) {
; CHECK-LABEL: zext_v8i16_v8i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    uunpklo z1.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v8i16_v8i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    str q0, [sp, #-64]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #22]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #64
; NONEON-NOSVE-NEXT:    ret
  %b = zext <8 x i16> %a to <8 x i32>
  store <8 x i32>%b, ptr %out
  ret void
}

define void @zext_v16i16_v16i32(ptr %in, ptr %out) {
; CHECK-LABEL: zext_v16i16_v16i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldp q1, q0, [x0]
; CHECK-NEXT:    add z0.h, z0.h, z0.h
; CHECK-NEXT:    add z1.h, z1.h, z1.h
; CHECK-NEXT:    uunpklo z2.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z3.s, z1.h
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    uunpklo z1.s, z1.h
; CHECK-NEXT:    stp q2, q0, [x1, #32]
; CHECK-NEXT:    stp q3, q1, [x1]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v16i16_v16i32:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #160
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    stp q1, q0, [sp]
; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #4]
; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #6]
; NONEON-NOSVE-NEXT:    ldrh w3, [sp, #2]
; NONEON-NOSVE-NEXT:    ldrh w5, [sp]
; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #12]
; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #14]
; NONEON-NOSVE-NEXT:    add w13, w13, w13
; NONEON-NOSVE-NEXT:    add w14, w14, w14
; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #10]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #46]
; NONEON-NOSVE-NEXT:    add w14, w3, w3
; NONEON-NOSVE-NEXT:    strh w13, [sp, #44]
; NONEON-NOSVE-NEXT:    add w13, w5, w5
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #42]
; NONEON-NOSVE-NEXT:    add w14, w4, w4
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w13, [sp, #40]
; NONEON-NOSVE-NEXT:    add w13, w2, w2
; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #22]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #38]
; NONEON-NOSVE-NEXT:    add w14, w0, w0
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    strh w13, [sp, #36]
; NONEON-NOSVE-NEXT:    add w13, w18, w18
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    strh w14, [sp, #34]
; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #28]
; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #30]
; NONEON-NOSVE-NEXT:    strh w13, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #18]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #20]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #50]
; NONEON-NOSVE-NEXT:    add w14, w17, w17
; NONEON-NOSVE-NEXT:    add w12, w12, w12
; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
; NONEON-NOSVE-NEXT:    add w13, w16, w16
; NONEON-NOSVE-NEXT:    add w11, w11, w11
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    add w10, w10, w10
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #78]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #76]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #62]
; NONEON-NOSVE-NEXT:    add w14, w15, w15
; NONEON-NOSVE-NEXT:    strh w13, [sp, #60]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #74]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #72]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #58]
; NONEON-NOSVE-NEXT:    strh w12, [sp, #56]
; NONEON-NOSVE-NEXT:    strh w11, [sp, #54]
; NONEON-NOSVE-NEXT:    strh w10, [sp, #52]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #70]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #68]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #66]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #64]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #94]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #92]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #96]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #90]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #88]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #86]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #84]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #82]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #80]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
; NONEON-NOSVE-NEXT:    add sp, sp, #160
; NONEON-NOSVE-NEXT:    ret
  %a = load <16 x i16>, ptr %in
  %b = add <16 x i16> %a, %a
  %c = zext <16 x i16> %b to <16 x i32>
  store <16 x i32> %c, ptr %out
  ret void
}

;
; zext i16 -> i64
;

define void @zext_v4i16_v4i64(<4 x i16> %a, ptr %out) {
; CHECK-LABEL: zext_v4i16_v4i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    uunpklo z1.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.d, z0.s
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v4i16_v4i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #80
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #40]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #72]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #64]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #32]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #56]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #48]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #48]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #80
; NONEON-NOSVE-NEXT:    ret
  %b = zext <4 x i16> %a to <4 x i64>
  store <4 x i64>%b, ptr %out
  ret void
}

define void @zext_v8i16_v8i64(<8 x i16> %a, ptr %out) {
; CHECK-LABEL: zext_v8i16_v8i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    uunpklo z1.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    uunpklo z2.d, z1.s
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    uunpklo z1.d, z1.s
; CHECK-NEXT:    uunpklo z3.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.d, z0.s
; CHECK-NEXT:    stp q2, q1, [x0]
; CHECK-NEXT:    stp q3, q0, [x0, #32]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v8i16_v8i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    str q0, [sp, #-160]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #30]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #18]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #22]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #88]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #152]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #144]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #80]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #136]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #128]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #72]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #128]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #120]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #112]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #64]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #104]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x0]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0, #32]
; NONEON-NOSVE-NEXT:    add sp, sp, #160
; NONEON-NOSVE-NEXT:    ret
  %b = zext <8 x i16> %a to <8 x i64>
  store <8 x i64>%b, ptr %out
  ret void
}

define void @zext_v16i16_v16i64(ptr %in, ptr %out) {
; CHECK-LABEL: zext_v16i16_v16i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldp q1, q0, [x0]
; CHECK-NEXT:    add z0.h, z0.h, z0.h
; CHECK-NEXT:    add z1.h, z1.h, z1.h
; CHECK-NEXT:    uunpklo z2.s, z0.h
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z3.s, z1.h
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    uunpklo z0.s, z0.h
; CHECK-NEXT:    uunpklo z1.s, z1.h
; CHECK-NEXT:    uunpklo z4.d, z2.s
; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
; CHECK-NEXT:    uunpklo z5.d, z3.s
; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
; CHECK-NEXT:    uunpklo z6.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z2.d, z2.s
; CHECK-NEXT:    uunpklo z7.d, z1.s
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    uunpklo z3.d, z3.s
; CHECK-NEXT:    uunpklo z0.d, z0.s
; CHECK-NEXT:    uunpklo z1.d, z1.s
; CHECK-NEXT:    stp q4, q2, [x1, #64]
; CHECK-NEXT:    stp q5, q3, [x1]
; CHECK-NEXT:    stp q6, q0, [x1, #96]
; CHECK-NEXT:    stp q7, q1, [x1, #32]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v16i16_v16i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #368
; NONEON-NOSVE-NEXT:    str x29, [sp, #352] // 8-byte Folded Spill
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 368
; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #268]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #260]
; NONEON-NOSVE-NEXT:    ldr x29, [sp, #352] // 8-byte Folded Reload
; NONEON-NOSVE-NEXT:    str wzr, [sp, #284]
; NONEON-NOSVE-NEXT:    stp q1, q0, [sp]
; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #4]
; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #6]
; NONEON-NOSVE-NEXT:    ldrh w3, [sp, #2]
; NONEON-NOSVE-NEXT:    ldrh w5, [sp]
; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #12]
; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #14]
; NONEON-NOSVE-NEXT:    add w13, w13, w13
; NONEON-NOSVE-NEXT:    add w14, w14, w14
; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #8]
; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #10]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #54]
; NONEON-NOSVE-NEXT:    add w14, w3, w3
; NONEON-NOSVE-NEXT:    strh w13, [sp, #52]
; NONEON-NOSVE-NEXT:    add w13, w5, w5
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #50]
; NONEON-NOSVE-NEXT:    add w14, w4, w4
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]
; NONEON-NOSVE-NEXT:    strh w13, [sp, #48]
; NONEON-NOSVE-NEXT:    add w13, w2, w2
; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #22]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #46]
; NONEON-NOSVE-NEXT:    add w14, w0, w0
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    strh w13, [sp, #44]
; NONEON-NOSVE-NEXT:    add w13, w18, w18
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    strh w14, [sp, #42]
; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #28]
; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #30]
; NONEON-NOSVE-NEXT:    strh w13, [sp, #40]
; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #16]
; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #18]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #40]
; NONEON-NOSVE-NEXT:    add w14, w17, w17
; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #20]
; NONEON-NOSVE-NEXT:    strh w9, [sp, #58]
; NONEON-NOSVE-NEXT:    add w12, w12, w12
; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
; NONEON-NOSVE-NEXT:    add w11, w11, w11
; NONEON-NOSVE-NEXT:    add w10, w10, w10
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #72]
; NONEON-NOSVE-NEXT:    add w13, w16, w16
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #82]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #80]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #70]
; NONEON-NOSVE-NEXT:    add w14, w15, w15
; NONEON-NOSVE-NEXT:    strh w13, [sp, #68]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #86]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #84]
; NONEON-NOSVE-NEXT:    strh w14, [sp, #66]
; NONEON-NOSVE-NEXT:    strh w12, [sp, #64]
; NONEON-NOSVE-NEXT:    strh w11, [sp, #62]
; NONEON-NOSVE-NEXT:    strh w10, [sp, #60]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #56]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #74]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #72]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #276]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #332]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #78]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #76]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #88]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #120]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #98]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #96]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #324]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #152]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #102]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #100]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #184]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #104]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #160]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #90]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #88]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #348]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #136]
; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #94]
; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #92]
; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #168]
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #152]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #144]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #184]
; NONEON-NOSVE-NEXT:    str d0, [sp, #360]
; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp, #136]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #340]
; NONEON-NOSVE-NEXT:    str w9, [sp, #264]
; NONEON-NOSVE-NEXT:    stp wzr, w8, [sp, #252]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #192]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]
; NONEON-NOSVE-NEXT:    str d2, [sp, #200]
; NONEON-NOSVE-NEXT:    str w9, [sp, #280]
; NONEON-NOSVE-NEXT:    str w8, [sp, #272]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #168]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #300]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #256]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #292]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #232]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #224]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #176]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #316]
; NONEON-NOSVE-NEXT:    str wzr, [sp, #308]
; NONEON-NOSVE-NEXT:    stp wzr, w9, [sp, #244]
; NONEON-NOSVE-NEXT:    str w8, [sp, #240]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #216]
; NONEON-NOSVE-NEXT:    ldp q3, q4, [sp, #224]
; NONEON-NOSVE-NEXT:    str w8, [sp, #320]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #364]
; NONEON-NOSVE-NEXT:    str w9, [sp, #328]
; NONEON-NOSVE-NEXT:    str w8, [sp, #344]
; NONEON-NOSVE-NEXT:    ldr w8, [sp, #360]
; NONEON-NOSVE-NEXT:    str w8, [sp, #336]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #200]
; NONEON-NOSVE-NEXT:    ldp q6, q7, [sp, #320]
; NONEON-NOSVE-NEXT:    str w9, [sp, #296]
; NONEON-NOSVE-NEXT:    str w8, [sp, #288]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #208]
; NONEON-NOSVE-NEXT:    str w9, [sp, #312]
; NONEON-NOSVE-NEXT:    str w8, [sp, #304]
; NONEON-NOSVE-NEXT:    ldp q5, q2, [sp, #288]
; NONEON-NOSVE-NEXT:    stp q0, q1, [x1]
; NONEON-NOSVE-NEXT:    stp q3, q4, [x1, #32]
; NONEON-NOSVE-NEXT:    stp q6, q7, [x1, #64]
; NONEON-NOSVE-NEXT:    stp q5, q2, [x1, #96]
; NONEON-NOSVE-NEXT:    add sp, sp, #368
; NONEON-NOSVE-NEXT:    ret
  %a = load <16 x i16>, ptr %in
  %b = add <16 x i16> %a, %a
  %c = zext <16 x i16> %b to <16 x i64>
  store <16 x i64> %c, ptr %out
  ret void
}

;
; zext i32 -> i64
;

define void @zext_v4i32_v4i64(<4 x i32> %a, ptr %out) {
; CHECK-LABEL: zext_v4i32_v4i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
; CHECK-NEXT:    uunpklo z1.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z0.d, z0.s
; CHECK-NEXT:    stp q1, q0, [x0]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v4i32_v4i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    str q0, [sp, #-64]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #16]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #24]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #56]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #48]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #16]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #40]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #32]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    add sp, sp, #64
; NONEON-NOSVE-NEXT:    ret
  %b = zext <4 x i32> %a to <4 x i64>
  store <4 x i64>%b, ptr %out
  ret void
}

define void @zext_v8i32_v8i64(ptr %in, ptr %out) {
; CHECK-LABEL: zext_v8i32_v8i64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldp q1, q0, [x0]
; CHECK-NEXT:    add z0.s, z0.s, z0.s
; CHECK-NEXT:    add z1.s, z1.s, z1.s
; CHECK-NEXT:    uunpklo z2.d, z0.s
; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
; CHECK-NEXT:    uunpklo z3.d, z1.s
; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
; CHECK-NEXT:    uunpklo z0.d, z0.s
; CHECK-NEXT:    uunpklo z1.d, z1.s
; CHECK-NEXT:    stp q2, q0, [x1, #32]
; CHECK-NEXT:    stp q3, q1, [x1]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: zext_v8i32_v8i64:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    sub sp, sp, #160
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160
; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT:    stp q1, q0, [sp]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp]
; NONEON-NOSVE-NEXT:    ldp w12, w13, [sp, #8]
; NONEON-NOSVE-NEXT:    ldp w14, w15, [sp, #16]
; NONEON-NOSVE-NEXT:    add w9, w9, w9
; NONEON-NOSVE-NEXT:    add w8, w8, w8
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
; NONEON-NOSVE-NEXT:    add w9, w13, w13
; NONEON-NOSVE-NEXT:    add w8, w12, w12
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
; NONEON-NOSVE-NEXT:    add w9, w15, w15
; NONEON-NOSVE-NEXT:    add w8, w14, w14
; NONEON-NOSVE-NEXT:    ldp w10, w11, [sp, #24]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
; NONEON-NOSVE-NEXT:    add w9, w11, w11
; NONEON-NOSVE-NEXT:    add w8, w10, w10
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #64]
; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #72]
; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #48]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #120]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #112]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #64]
; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #104]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #96]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #88]
; NONEON-NOSVE-NEXT:    ldp q3, q2, [sp, #96]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #152]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #144]
; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #80]
; NONEON-NOSVE-NEXT:    stp w9, wzr, [sp, #136]
; NONEON-NOSVE-NEXT:    stp w8, wzr, [sp, #128]
; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]
; NONEON-NOSVE-NEXT:    stp q2, q3, [x1]
; NONEON-NOSVE-NEXT:    stp q1, q0, [x1, #32]
; NONEON-NOSVE-NEXT:    add sp, sp, #160
; NONEON-NOSVE-NEXT:    ret
  %a = load <8 x i32>, ptr %in
  %b = add <8 x i32> %a, %a
  %c = zext <8 x i32> %b to <8 x i64>
  store <8 x i64> %c, ptr %out
  ret void
}

define void @extend_and_mul(i32 %0, <2 x i64> %1, ptr %2) {
; SVE-LABEL: extend_and_mul:
; SVE:       // %bb.0:
; SVE-NEXT:    mov z1.s, w0
; SVE-NEXT:    ptrue p0.d, vl2
; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
; SVE-NEXT:    uunpklo z1.d, z1.s
; SVE-NEXT:    mul z0.d, p0/m, z0.d, z1.d
; SVE-NEXT:    str q0, [x1]
; SVE-NEXT:    ret
;
; SVE2-LABEL: extend_and_mul:
; SVE2:       // %bb.0:
; SVE2-NEXT:    mov z1.s, w0
; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
; SVE2-NEXT:    uunpklo z1.d, z1.s
; SVE2-NEXT:    mul z0.d, z1.d, z0.d
; SVE2-NEXT:    str q0, [x1]
; SVE2-NEXT:    ret
;
; NONEON-NOSVE-LABEL: extend_and_mul:
; NONEON-NOSVE:       // %bb.0:
; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT:    ldr x8, [sp, #8]
; NONEON-NOSVE-NEXT:    mov w9, w0
; NONEON-NOSVE-NEXT:    mul x10, x9, x8
; NONEON-NOSVE-NEXT:    ldr x8, [sp]
; NONEON-NOSVE-NEXT:    mul x8, x9, x8
; NONEON-NOSVE-NEXT:    stp x8, x10, [sp, #16]
; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT:    str q0, [x1]
; NONEON-NOSVE-NEXT:    add sp, sp, #32
; NONEON-NOSVE-NEXT:    ret
  %broadcast.splatinsert2 = insertelement <2 x i32> poison, i32 %0, i64 0
  %broadcast.splat3 = shufflevector <2 x i32> %broadcast.splatinsert2, <2 x i32> poison, <2 x i32> zeroinitializer
  %4 = zext <2 x i32> %broadcast.splat3 to <2 x i64>
  %5 = mul <2 x i64> %4, %1
  store <2 x i64> %5, ptr %2, align 2
  ret void
}

define void @extend_no_mul(i32 %0, <2 x i64> %1, ptr %2) {
; CHECK-LABEL: extend_no_mul:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    mov z0.d, x8
; CHECK-NEXT:    str q0, [x1]
; CHECK-NEXT:    ret
;
; NONEON-NOSVE-LABEL: extend_no_mul:
; NONEON-NOSVE:       // %bb.0: // %entry
; NONEON-NOSVE-NEXT:    mov w8, w0
; NONEON-NOSVE-NEXT:    stp x8, x8, [sp, #-16]!
; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
; NONEON-NOSVE-NEXT:    ldr q0, [sp]
; NONEON-NOSVE-NEXT:    str q0, [x1]
; NONEON-NOSVE-NEXT:    add sp, sp, #16
; NONEON-NOSVE-NEXT:    ret
entry:
  %broadcast.splatinsert2 = insertelement <2 x i32> poison, i32 %0, i64 0
  %broadcast.splat3 = shufflevector <2 x i32> %broadcast.splatinsert2, <2 x i32> poison, <2 x i32> zeroinitializer
  %3 = zext <2 x i32> %broadcast.splat3 to <2 x i64>
  store <2 x i64> %3, ptr %2, align 2
  ret void
}