; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI

define i8 @rotl_i8(i8 %a, i8 %c) {
; CHECK-SD-LABEL: rotl_i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    neg w8, w1
; CHECK-SD-NEXT:    and w9, w0, #0xff
; CHECK-SD-NEXT:    and w10, w1, #0x7
; CHECK-SD-NEXT:    and w8, w8, #0x7
; CHECK-SD-NEXT:    lsl w10, w0, w10
; CHECK-SD-NEXT:    lsr w8, w9, w8
; CHECK-SD-NEXT:    orr w0, w10, w8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov x8, xzr
; CHECK-GI-NEXT:    neg w9, w1
; CHECK-GI-NEXT:    and w10, w0, #0xff
; CHECK-GI-NEXT:    sub x8, x8, w9, uxtb
; CHECK-GI-NEXT:    and x9, x9, #0x7
; CHECK-GI-NEXT:    lsr w9, w10, w9
; CHECK-GI-NEXT:    and x8, x8, #0x7
; CHECK-GI-NEXT:    lsl w8, w0, w8
; CHECK-GI-NEXT:    orr w0, w9, w8
; CHECK-GI-NEXT:    ret
entry:
  %d = call i8 @llvm.fshl(i8 %a, i8 %a, i8 %c)
  ret i8 %d
}

define i8 @rotr_i8(i8 %a, i8 %c) {
; CHECK-SD-LABEL: rotr_i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    neg w8, w1
; CHECK-SD-NEXT:    and w9, w0, #0xff
; CHECK-SD-NEXT:    and w10, w1, #0x7
; CHECK-SD-NEXT:    and w8, w8, #0x7
; CHECK-SD-NEXT:    lsr w9, w9, w10
; CHECK-SD-NEXT:    lsl w8, w0, w8
; CHECK-SD-NEXT:    orr w0, w9, w8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov x8, xzr
; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
; CHECK-GI-NEXT:    and x9, x1, #0x7
; CHECK-GI-NEXT:    and w10, w0, #0xff
; CHECK-GI-NEXT:    sub x8, x8, w1, uxtb
; CHECK-GI-NEXT:    lsr w9, w10, w9
; CHECK-GI-NEXT:    and x8, x8, #0x7
; CHECK-GI-NEXT:    lsl w8, w0, w8
; CHECK-GI-NEXT:    orr w0, w9, w8
; CHECK-GI-NEXT:    ret
entry:
  %d = call i8 @llvm.fshr(i8 %a, i8 %a, i8 %c)
  ret i8 %d
}

define i16 @rotl_i16(i16 %a, i16 %c) {
; CHECK-SD-LABEL: rotl_i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    neg w8, w1
; CHECK-SD-NEXT:    and w9, w0, #0xffff
; CHECK-SD-NEXT:    and w10, w1, #0xf
; CHECK-SD-NEXT:    and w8, w8, #0xf
; CHECK-SD-NEXT:    lsl w10, w0, w10
; CHECK-SD-NEXT:    lsr w8, w9, w8
; CHECK-SD-NEXT:    orr w0, w10, w8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov x8, xzr
; CHECK-GI-NEXT:    neg w9, w1
; CHECK-GI-NEXT:    and w10, w0, #0xffff
; CHECK-GI-NEXT:    sub x8, x8, w9, uxth
; CHECK-GI-NEXT:    and x9, x9, #0xf
; CHECK-GI-NEXT:    lsr w9, w10, w9
; CHECK-GI-NEXT:    and x8, x8, #0xf
; CHECK-GI-NEXT:    lsl w8, w0, w8
; CHECK-GI-NEXT:    orr w0, w9, w8
; CHECK-GI-NEXT:    ret
entry:
  %d = call i16 @llvm.fshl(i16 %a, i16 %a, i16 %c)
  ret i16 %d
}

define i16 @rotr_i16(i16 %a, i16 %c) {
; CHECK-SD-LABEL: rotr_i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    neg w8, w1
; CHECK-SD-NEXT:    and w9, w0, #0xffff
; CHECK-SD-NEXT:    and w10, w1, #0xf
; CHECK-SD-NEXT:    and w8, w8, #0xf
; CHECK-SD-NEXT:    lsr w9, w9, w10
; CHECK-SD-NEXT:    lsl w8, w0, w8
; CHECK-SD-NEXT:    orr w0, w9, w8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov x8, xzr
; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
; CHECK-GI-NEXT:    and x9, x1, #0xf
; CHECK-GI-NEXT:    and w10, w0, #0xffff
; CHECK-GI-NEXT:    sub x8, x8, w1, uxth
; CHECK-GI-NEXT:    lsr w9, w10, w9
; CHECK-GI-NEXT:    and x8, x8, #0xf
; CHECK-GI-NEXT:    lsl w8, w0, w8
; CHECK-GI-NEXT:    orr w0, w9, w8
; CHECK-GI-NEXT:    ret
entry:
  %d = call i16 @llvm.fshr(i16 %a, i16 %a, i16 %c)
  ret i16 %d
}

define i32 @rotl_i32(i32 %a, i32 %c) {
; CHECK-LABEL: rotl_i32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    neg w8, w1
; CHECK-NEXT:    ror w0, w0, w8
; CHECK-NEXT:    ret
entry:
  %d = call i32 @llvm.fshl(i32 %a, i32 %a, i32 %c)
  ret i32 %d
}

define i32 @rotr_i32(i32 %a, i32 %c) {
; CHECK-LABEL: rotr_i32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ror w0, w0, w1
; CHECK-NEXT:    ret
entry:
  %d = call i32 @llvm.fshr(i32 %a, i32 %a, i32 %c)
  ret i32 %d
}

define i64 @rotl_i64(i64 %a, i64 %c) {
; CHECK-SD-LABEL: rotl_i64:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    neg w8, w1
; CHECK-SD-NEXT:    ror x0, x0, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_i64:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    neg x8, x1
; CHECK-GI-NEXT:    ror x0, x0, x8
; CHECK-GI-NEXT:    ret
entry:
  %d = call i64 @llvm.fshl(i64 %a, i64 %a, i64 %c)
  ret i64 %d
}

define i64 @rotr_i64(i64 %a, i64 %c) {
; CHECK-LABEL: rotr_i64:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ror x0, x0, x1
; CHECK-NEXT:    ret
entry:
  %d = call i64 @llvm.fshr(i64 %a, i64 %a, i64 %c)
  ret i64 %d
}

define i128 @rotl_i128(i128 %a, i128 %c) {
; CHECK-SD-LABEL: rotl_i128:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    tst x2, #0x40
; CHECK-SD-NEXT:    mvn w12, w2
; CHECK-SD-NEXT:    csel x8, x1, x0, ne
; CHECK-SD-NEXT:    csel x9, x0, x1, ne
; CHECK-SD-NEXT:    lsr x10, x9, #1
; CHECK-SD-NEXT:    lsr x11, x8, #1
; CHECK-SD-NEXT:    lsl x8, x8, x2
; CHECK-SD-NEXT:    lsl x9, x9, x2
; CHECK-SD-NEXT:    lsr x10, x10, x12
; CHECK-SD-NEXT:    lsr x11, x11, x12
; CHECK-SD-NEXT:    orr x0, x8, x10
; CHECK-SD-NEXT:    orr x1, x9, x11
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_i128:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #64 // =0x40
; CHECK-GI-NEXT:    and x9, x2, #0x7f
; CHECK-GI-NEXT:    neg x10, x2
; CHECK-GI-NEXT:    sub x12, x8, x9
; CHECK-GI-NEXT:    sub x11, x9, #64
; CHECK-GI-NEXT:    lsl x14, x1, x9
; CHECK-GI-NEXT:    lsr x12, x0, x12
; CHECK-GI-NEXT:    lsl x13, x0, x9
; CHECK-GI-NEXT:    cmp x9, #64
; CHECK-GI-NEXT:    and x9, x10, #0x7f
; CHECK-GI-NEXT:    lsl x11, x0, x11
; CHECK-GI-NEXT:    orr x12, x12, x14
; CHECK-GI-NEXT:    sub x8, x8, x9
; CHECK-GI-NEXT:    sub x14, x9, #64
; CHECK-GI-NEXT:    csel x11, x12, x11, lo
; CHECK-GI-NEXT:    lsr x12, x0, x9
; CHECK-GI-NEXT:    lsl x8, x1, x8
; CHECK-GI-NEXT:    csel x13, x13, xzr, lo
; CHECK-GI-NEXT:    tst x2, #0x7f
; CHECK-GI-NEXT:    lsr x14, x1, x14
; CHECK-GI-NEXT:    csel x11, x1, x11, eq
; CHECK-GI-NEXT:    orr x8, x12, x8
; CHECK-GI-NEXT:    cmp x9, #64
; CHECK-GI-NEXT:    lsr x12, x1, x9
; CHECK-GI-NEXT:    csel x8, x8, x14, lo
; CHECK-GI-NEXT:    tst x10, #0x7f
; CHECK-GI-NEXT:    csel x8, x0, x8, eq
; CHECK-GI-NEXT:    cmp x9, #64
; CHECK-GI-NEXT:    csel x9, x12, xzr, lo
; CHECK-GI-NEXT:    orr x0, x13, x8
; CHECK-GI-NEXT:    orr x1, x11, x9
; CHECK-GI-NEXT:    ret
entry:
  %d = call i128 @llvm.fshl(i128 %a, i128 %a, i128 %c)
  ret i128 %d
}

define i128 @rotr_i128(i128 %a, i128 %c) {
; CHECK-SD-LABEL: rotr_i128:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    tst x2, #0x40
; CHECK-SD-NEXT:    mvn w12, w2
; CHECK-SD-NEXT:    csel x8, x0, x1, eq
; CHECK-SD-NEXT:    csel x9, x1, x0, eq
; CHECK-SD-NEXT:    lsl x10, x9, #1
; CHECK-SD-NEXT:    lsl x11, x8, #1
; CHECK-SD-NEXT:    lsr x8, x8, x2
; CHECK-SD-NEXT:    lsr x9, x9, x2
; CHECK-SD-NEXT:    lsl x10, x10, x12
; CHECK-SD-NEXT:    lsl x11, x11, x12
; CHECK-SD-NEXT:    orr x0, x10, x8
; CHECK-SD-NEXT:    orr x1, x11, x9
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_i128:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #64 // =0x40
; CHECK-GI-NEXT:    and x9, x2, #0x7f
; CHECK-GI-NEXT:    neg x13, x2
; CHECK-GI-NEXT:    sub x10, x8, x9
; CHECK-GI-NEXT:    sub x11, x9, #64
; CHECK-GI-NEXT:    lsr x12, x0, x9
; CHECK-GI-NEXT:    lsl x10, x1, x10
; CHECK-GI-NEXT:    lsr x11, x1, x11
; CHECK-GI-NEXT:    and x14, x13, #0x7f
; CHECK-GI-NEXT:    cmp x9, #64
; CHECK-GI-NEXT:    sub x8, x8, x14
; CHECK-GI-NEXT:    lsl x15, x1, x14
; CHECK-GI-NEXT:    orr x10, x12, x10
; CHECK-GI-NEXT:    lsr x12, x1, x9
; CHECK-GI-NEXT:    lsr x8, x0, x8
; CHECK-GI-NEXT:    csel x10, x10, x11, lo
; CHECK-GI-NEXT:    sub x11, x14, #64
; CHECK-GI-NEXT:    tst x2, #0x7f
; CHECK-GI-NEXT:    csel x10, x0, x10, eq
; CHECK-GI-NEXT:    cmp x9, #64
; CHECK-GI-NEXT:    lsl x9, x0, x14
; CHECK-GI-NEXT:    lsl x11, x0, x11
; CHECK-GI-NEXT:    csel x12, x12, xzr, lo
; CHECK-GI-NEXT:    orr x8, x8, x15
; CHECK-GI-NEXT:    cmp x14, #64
; CHECK-GI-NEXT:    csel x9, x9, xzr, lo
; CHECK-GI-NEXT:    csel x8, x8, x11, lo
; CHECK-GI-NEXT:    tst x13, #0x7f
; CHECK-GI-NEXT:    csel x8, x1, x8, eq
; CHECK-GI-NEXT:    orr x0, x10, x9
; CHECK-GI-NEXT:    orr x1, x12, x8
; CHECK-GI-NEXT:    ret
entry:
  %d = call i128 @llvm.fshr(i128 %a, i128 %a, i128 %c)
  ret i128 %d
}

define i8 @fshl_i8(i8 %a, i8 %b, i8 %c) {
; CHECK-SD-LABEL: fshl_i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    and w8, w2, #0x7
; CHECK-SD-NEXT:    ubfiz w9, w1, #23, #8
; CHECK-SD-NEXT:    mvn w10, w8
; CHECK-SD-NEXT:    lsl w8, w0, w8
; CHECK-SD-NEXT:    lsr w9, w9, w10
; CHECK-SD-NEXT:    orr w0, w8, w9
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #7 // =0x7
; CHECK-GI-NEXT:    ubfx w9, w1, #1, #7
; CHECK-GI-NEXT:    and w10, w2, #0x7
; CHECK-GI-NEXT:    bic w8, w8, w2
; CHECK-GI-NEXT:    lsl w10, w0, w10
; CHECK-GI-NEXT:    lsr w8, w9, w8
; CHECK-GI-NEXT:    orr w0, w10, w8
; CHECK-GI-NEXT:    ret
entry:
  %d = call i8 @llvm.fshl(i8 %a, i8 %b, i8 %c)
  ret i8 %d
}

define i8 @fshr_i8(i8 %a, i8 %b, i8 %c) {
; CHECK-SD-LABEL: fshr_i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mov w8, #24 // =0x18
; CHECK-SD-NEXT:    lsl w9, w1, #24
; CHECK-SD-NEXT:    mvn w10, w2
; CHECK-SD-NEXT:    bfxil w8, w2, #0, #3
; CHECK-SD-NEXT:    lsl w11, w0, #1
; CHECK-SD-NEXT:    and x10, x10, #0x7
; CHECK-SD-NEXT:    lsr w8, w9, w8
; CHECK-SD-NEXT:    lsl w9, w11, w10
; CHECK-SD-NEXT:    orr w0, w9, w8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #7 // =0x7
; CHECK-GI-NEXT:    lsl w9, w0, #1
; CHECK-GI-NEXT:    and w10, w2, #0x7
; CHECK-GI-NEXT:    bic w8, w8, w2
; CHECK-GI-NEXT:    and w11, w1, #0xff
; CHECK-GI-NEXT:    lsl w8, w9, w8
; CHECK-GI-NEXT:    lsr w9, w11, w10
; CHECK-GI-NEXT:    orr w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %d = call i8 @llvm.fshr(i8 %a, i8 %b, i8 %c)
  ret i8 %d
}

define i16 @fshl_i16(i16 %a, i16 %b, i16 %c) {
; CHECK-SD-LABEL: fshl_i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    and w8, w2, #0xf
; CHECK-SD-NEXT:    ubfiz w9, w1, #15, #16
; CHECK-SD-NEXT:    mvn w10, w8
; CHECK-SD-NEXT:    lsl w8, w0, w8
; CHECK-SD-NEXT:    lsr w9, w9, w10
; CHECK-SD-NEXT:    orr w0, w8, w9
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #15 // =0xf
; CHECK-GI-NEXT:    ubfx w9, w1, #1, #15
; CHECK-GI-NEXT:    and w10, w2, #0xf
; CHECK-GI-NEXT:    bic w8, w8, w2
; CHECK-GI-NEXT:    lsl w10, w0, w10
; CHECK-GI-NEXT:    lsr w8, w9, w8
; CHECK-GI-NEXT:    orr w0, w10, w8
; CHECK-GI-NEXT:    ret
entry:
  %d = call i16 @llvm.fshl(i16 %a, i16 %b, i16 %c)
  ret i16 %d
}

define i16 @fshr_i16(i16 %a, i16 %b, i16 %c) {
; CHECK-SD-LABEL: fshr_i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mov w8, #16 // =0x10
; CHECK-SD-NEXT:    lsl w9, w1, #16
; CHECK-SD-NEXT:    mvn w10, w2
; CHECK-SD-NEXT:    bfxil w8, w2, #0, #4
; CHECK-SD-NEXT:    lsl w11, w0, #1
; CHECK-SD-NEXT:    and x10, x10, #0xf
; CHECK-SD-NEXT:    lsr w8, w9, w8
; CHECK-SD-NEXT:    lsl w9, w11, w10
; CHECK-SD-NEXT:    orr w0, w9, w8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #15 // =0xf
; CHECK-GI-NEXT:    lsl w9, w0, #1
; CHECK-GI-NEXT:    and w10, w2, #0xf
; CHECK-GI-NEXT:    bic w8, w8, w2
; CHECK-GI-NEXT:    and w11, w1, #0xffff
; CHECK-GI-NEXT:    lsl w8, w9, w8
; CHECK-GI-NEXT:    lsr w9, w11, w10
; CHECK-GI-NEXT:    orr w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %d = call i16 @llvm.fshr(i16 %a, i16 %b, i16 %c)
  ret i16 %d
}

define i32 @fshl_i32(i32 %a, i32 %b, i32 %c) {
; CHECK-SD-LABEL: fshl_i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    lsr w8, w1, #1
; CHECK-SD-NEXT:    mvn w9, w2
; CHECK-SD-NEXT:    lsl w10, w0, w2
; CHECK-SD-NEXT:    lsr w8, w8, w9
; CHECK-SD-NEXT:    orr w0, w10, w8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #31 // =0x1f
; CHECK-GI-NEXT:    lsr w9, w1, #1
; CHECK-GI-NEXT:    and w10, w2, #0x1f
; CHECK-GI-NEXT:    bic w8, w8, w2
; CHECK-GI-NEXT:    lsl w10, w0, w10
; CHECK-GI-NEXT:    lsr w8, w9, w8
; CHECK-GI-NEXT:    orr w0, w10, w8
; CHECK-GI-NEXT:    ret
entry:
  %d = call i32 @llvm.fshl(i32 %a, i32 %b, i32 %c)
  ret i32 %d
}

define i32 @fshr_i32(i32 %a, i32 %b, i32 %c) {
; CHECK-SD-LABEL: fshr_i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    lsl w8, w0, #1
; CHECK-SD-NEXT:    mvn w9, w2
; CHECK-SD-NEXT:    lsr w10, w1, w2
; CHECK-SD-NEXT:    lsl w8, w8, w9
; CHECK-SD-NEXT:    orr w0, w8, w10
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #31 // =0x1f
; CHECK-GI-NEXT:    lsl w9, w0, #1
; CHECK-GI-NEXT:    and w10, w2, #0x1f
; CHECK-GI-NEXT:    bic w8, w8, w2
; CHECK-GI-NEXT:    lsl w8, w9, w8
; CHECK-GI-NEXT:    lsr w9, w1, w10
; CHECK-GI-NEXT:    orr w0, w8, w9
; CHECK-GI-NEXT:    ret
entry:
  %d = call i32 @llvm.fshr(i32 %a, i32 %b, i32 %c)
  ret i32 %d
}

define i64 @fshl_i64(i64 %a, i64 %b, i64 %c) {
; CHECK-SD-LABEL: fshl_i64:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    lsr x8, x1, #1
; CHECK-SD-NEXT:    mvn w9, w2
; CHECK-SD-NEXT:    lsl x10, x0, x2
; CHECK-SD-NEXT:    lsr x8, x8, x9
; CHECK-SD-NEXT:    orr x0, x10, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_i64:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #63 // =0x3f
; CHECK-GI-NEXT:    lsr x9, x1, #1
; CHECK-GI-NEXT:    and x10, x2, #0x3f
; CHECK-GI-NEXT:    bic x8, x8, x2
; CHECK-GI-NEXT:    lsl x10, x0, x10
; CHECK-GI-NEXT:    lsr x8, x9, x8
; CHECK-GI-NEXT:    orr x0, x10, x8
; CHECK-GI-NEXT:    ret
entry:
  %d = call i64 @llvm.fshl(i64 %a, i64 %b, i64 %c)
  ret i64 %d
}

define i64 @fshr_i64(i64 %a, i64 %b, i64 %c) {
; CHECK-SD-LABEL: fshr_i64:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    lsl x8, x0, #1
; CHECK-SD-NEXT:    mvn w9, w2
; CHECK-SD-NEXT:    lsr x10, x1, x2
; CHECK-SD-NEXT:    lsl x8, x8, x9
; CHECK-SD-NEXT:    orr x0, x8, x10
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_i64:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #63 // =0x3f
; CHECK-GI-NEXT:    lsl x9, x0, #1
; CHECK-GI-NEXT:    and x10, x2, #0x3f
; CHECK-GI-NEXT:    bic x8, x8, x2
; CHECK-GI-NEXT:    lsl x8, x9, x8
; CHECK-GI-NEXT:    lsr x9, x1, x10
; CHECK-GI-NEXT:    orr x0, x8, x9
; CHECK-GI-NEXT:    ret
entry:
  %d = call i64 @llvm.fshr(i64 %a, i64 %b, i64 %c)
  ret i64 %d
}

define i128 @fshl_i128(i128 %a, i128 %b, i128 %c) {
; CHECK-SD-LABEL: fshl_i128:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    tst x4, #0x40
; CHECK-SD-NEXT:    mvn w11, w4
; CHECK-SD-NEXT:    csel x8, x3, x0, ne
; CHECK-SD-NEXT:    csel x9, x2, x3, ne
; CHECK-SD-NEXT:    csel x12, x0, x1, ne
; CHECK-SD-NEXT:    lsr x9, x9, #1
; CHECK-SD-NEXT:    lsr x10, x8, #1
; CHECK-SD-NEXT:    lsl x8, x8, x4
; CHECK-SD-NEXT:    lsl x12, x12, x4
; CHECK-SD-NEXT:    lsr x9, x9, x11
; CHECK-SD-NEXT:    lsr x10, x10, x11
; CHECK-SD-NEXT:    orr x0, x8, x9
; CHECK-SD-NEXT:    orr x1, x12, x10
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_i128:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    and x9, x4, #0x7f
; CHECK-GI-NEXT:    mov w10, #64 // =0x40
; CHECK-GI-NEXT:    lsl x14, x3, #63
; CHECK-GI-NEXT:    sub x12, x10, x9
; CHECK-GI-NEXT:    lsl x13, x1, x9
; CHECK-GI-NEXT:    mov w8, #127 // =0x7f
; CHECK-GI-NEXT:    lsr x12, x0, x12
; CHECK-GI-NEXT:    bic x8, x8, x4
; CHECK-GI-NEXT:    sub x15, x9, #64
; CHECK-GI-NEXT:    cmp x9, #64
; CHECK-GI-NEXT:    lsl x9, x0, x9
; CHECK-GI-NEXT:    lsl x15, x0, x15
; CHECK-GI-NEXT:    orr x12, x12, x13
; CHECK-GI-NEXT:    orr x13, x14, x2, lsr #1
; CHECK-GI-NEXT:    lsr x14, x3, #1
; CHECK-GI-NEXT:    sub x10, x10, x8
; CHECK-GI-NEXT:    sub x16, x8, #64
; CHECK-GI-NEXT:    csel x9, x9, xzr, lo
; CHECK-GI-NEXT:    lsr x17, x13, x8
; CHECK-GI-NEXT:    lsl x10, x14, x10
; CHECK-GI-NEXT:    csel x12, x12, x15, lo
; CHECK-GI-NEXT:    tst x4, #0x7f
; CHECK-GI-NEXT:    lsr x15, x14, x16
; CHECK-GI-NEXT:    mvn x11, x4
; CHECK-GI-NEXT:    csel x12, x1, x12, eq
; CHECK-GI-NEXT:    orr x10, x17, x10
; CHECK-GI-NEXT:    cmp x8, #64
; CHECK-GI-NEXT:    lsr x14, x14, x8
; CHECK-GI-NEXT:    csel x10, x10, x15, lo
; CHECK-GI-NEXT:    tst x11, #0x7f
; CHECK-GI-NEXT:    csel x10, x13, x10, eq
; CHECK-GI-NEXT:    cmp x8, #64
; CHECK-GI-NEXT:    csel x8, x14, xzr, lo
; CHECK-GI-NEXT:    orr x0, x9, x10
; CHECK-GI-NEXT:    orr x1, x12, x8
; CHECK-GI-NEXT:    ret
entry:
  %d = call i128 @llvm.fshl(i128 %a, i128 %b, i128 %c)
  ret i128 %d
}

define i128 @fshr_i128(i128 %a, i128 %b, i128 %c) {
; CHECK-SD-LABEL: fshr_i128:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    tst x4, #0x40
; CHECK-SD-NEXT:    mvn w12, w4
; CHECK-SD-NEXT:    csel x8, x3, x0, eq
; CHECK-SD-NEXT:    csel x9, x0, x1, eq
; CHECK-SD-NEXT:    csel x10, x2, x3, eq
; CHECK-SD-NEXT:    lsl x11, x8, #1
; CHECK-SD-NEXT:    lsl x9, x9, #1
; CHECK-SD-NEXT:    lsr x10, x10, x4
; CHECK-SD-NEXT:    lsr x8, x8, x4
; CHECK-SD-NEXT:    lsl x11, x11, x12
; CHECK-SD-NEXT:    lsl x9, x9, x12
; CHECK-SD-NEXT:    orr x0, x11, x10
; CHECK-SD-NEXT:    orr x1, x9, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_i128:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    lsr x8, x0, #63
; CHECK-GI-NEXT:    mov w9, #127 // =0x7f
; CHECK-GI-NEXT:    mov w10, #64 // =0x40
; CHECK-GI-NEXT:    bic x9, x9, x4
; CHECK-GI-NEXT:    lsl x11, x0, #1
; CHECK-GI-NEXT:    and x12, x4, #0x7f
; CHECK-GI-NEXT:    orr x8, x8, x1, lsl #1
; CHECK-GI-NEXT:    sub x14, x10, x9
; CHECK-GI-NEXT:    sub x17, x9, #64
; CHECK-GI-NEXT:    lsl x15, x11, x9
; CHECK-GI-NEXT:    lsr x14, x11, x14
; CHECK-GI-NEXT:    cmp x9, #64
; CHECK-GI-NEXT:    lsl x16, x8, x9
; CHECK-GI-NEXT:    sub x9, x10, x12
; CHECK-GI-NEXT:    lsl x10, x11, x17
; CHECK-GI-NEXT:    mvn x13, x4
; CHECK-GI-NEXT:    csel x11, x15, xzr, lo
; CHECK-GI-NEXT:    sub x15, x12, #64
; CHECK-GI-NEXT:    orr x14, x14, x16
; CHECK-GI-NEXT:    lsr x16, x2, x12
; CHECK-GI-NEXT:    lsl x9, x3, x9
; CHECK-GI-NEXT:    csel x10, x14, x10, lo
; CHECK-GI-NEXT:    tst x13, #0x7f
; CHECK-GI-NEXT:    lsr x13, x3, x15
; CHECK-GI-NEXT:    csel x8, x8, x10, eq
; CHECK-GI-NEXT:    orr x9, x16, x9
; CHECK-GI-NEXT:    cmp x12, #64
; CHECK-GI-NEXT:    lsr x10, x3, x12
; CHECK-GI-NEXT:    csel x9, x9, x13, lo
; CHECK-GI-NEXT:    tst x4, #0x7f
; CHECK-GI-NEXT:    csel x9, x2, x9, eq
; CHECK-GI-NEXT:    cmp x12, #64
; CHECK-GI-NEXT:    csel x10, x10, xzr, lo
; CHECK-GI-NEXT:    orr x0, x11, x9
; CHECK-GI-NEXT:    orr x1, x8, x10
; CHECK-GI-NEXT:    ret
entry:
  %d = call i128 @llvm.fshr(i128 %a, i128 %b, i128 %c)
  ret i128 %d
}

define i8 @rotl_i8_c(i8 %a) {
; CHECK-LABEL: rotl_i8_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ubfx w8, w0, #5, #3
; CHECK-NEXT:    orr w0, w8, w0, lsl #3
; CHECK-NEXT:    ret
entry:
  %d = call i8 @llvm.fshl(i8 %a, i8 %a, i8 3)
  ret i8 %d
}

define i8 @rotr_i8_c(i8 %a) {
; CHECK-SD-LABEL: rotr_i8_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    lsl w8, w0, #5
; CHECK-SD-NEXT:    bfxil w8, w0, #3, #5
; CHECK-SD-NEXT:    mov w0, w8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_i8_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ubfx w8, w0, #3, #5
; CHECK-GI-NEXT:    orr w0, w8, w0, lsl #5
; CHECK-GI-NEXT:    ret
entry:
  %d = call i8 @llvm.fshr(i8 %a, i8 %a, i8 3)
  ret i8 %d
}

define i16 @rotl_i16_c(i16 %a) {
; CHECK-LABEL: rotl_i16_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ubfx w8, w0, #13, #3
; CHECK-NEXT:    orr w0, w8, w0, lsl #3
; CHECK-NEXT:    ret
entry:
  %d = call i16 @llvm.fshl(i16 %a, i16 %a, i16 3)
  ret i16 %d
}

define i16 @rotr_i16_c(i16 %a) {
; CHECK-SD-LABEL: rotr_i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    lsl w8, w0, #13
; CHECK-SD-NEXT:    bfxil w8, w0, #3, #13
; CHECK-SD-NEXT:    mov w0, w8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ubfx w8, w0, #3, #13
; CHECK-GI-NEXT:    orr w0, w8, w0, lsl #13
; CHECK-GI-NEXT:    ret
entry:
  %d = call i16 @llvm.fshr(i16 %a, i16 %a, i16 3)
  ret i16 %d
}

define i32 @rotl_i32_c(i32 %a) {
; CHECK-LABEL: rotl_i32_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ror w0, w0, #29
; CHECK-NEXT:    ret
entry:
  %d = call i32 @llvm.fshl(i32 %a, i32 %a, i32 3)
  ret i32 %d
}

define i32 @rotr_i32_c(i32 %a) {
; CHECK-LABEL: rotr_i32_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ror w0, w0, #3
; CHECK-NEXT:    ret
entry:
  %d = call i32 @llvm.fshr(i32 %a, i32 %a, i32 3)
  ret i32 %d
}

define i64 @rotl_i64_c(i64 %a) {
; CHECK-LABEL: rotl_i64_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ror x0, x0, #61
; CHECK-NEXT:    ret
entry:
  %d = call i64 @llvm.fshl(i64 %a, i64 %a, i64 3)
  ret i64 %d
}

define i64 @rotr_i64_c(i64 %a) {
; CHECK-LABEL: rotr_i64_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ror x0, x0, #3
; CHECK-NEXT:    ret
entry:
  %d = call i64 @llvm.fshr(i64 %a, i64 %a, i64 3)
  ret i64 %d
}

define i128 @rotl_i128_c(i128 %a) {
; CHECK-SD-LABEL: rotl_i128_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    extr x8, x0, x1, #61
; CHECK-SD-NEXT:    extr x1, x1, x0, #61
; CHECK-SD-NEXT:    mov x0, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_i128_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    lsr x8, x0, #61
; CHECK-GI-NEXT:    lsr x9, x1, #61
; CHECK-GI-NEXT:    orr x1, x8, x1, lsl #3
; CHECK-GI-NEXT:    orr x0, x9, x0, lsl #3
; CHECK-GI-NEXT:    ret
entry:
  %d = call i128 @llvm.fshl(i128 %a, i128 %a, i128 3)
  ret i128 %d
}

define i128 @rotr_i128_c(i128 %a) {
; CHECK-SD-LABEL: rotr_i128_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    extr x8, x1, x0, #3
; CHECK-SD-NEXT:    extr x1, x0, x1, #3
; CHECK-SD-NEXT:    mov x0, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_i128_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    lsl x8, x1, #61
; CHECK-GI-NEXT:    lsl x9, x0, #61
; CHECK-GI-NEXT:    orr x0, x8, x0, lsr #3
; CHECK-GI-NEXT:    orr x1, x9, x1, lsr #3
; CHECK-GI-NEXT:    ret
entry:
  %d = call i128 @llvm.fshr(i128 %a, i128 %a, i128 3)
  ret i128 %d
}

define i8 @fshl_i8_c(i8 %a, i8 %b) {
; CHECK-SD-LABEL: fshl_i8_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    lsl w8, w1, #24
; CHECK-SD-NEXT:    extr w0, w0, w8, #29
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_i8_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ubfx w8, w1, #5, #3
; CHECK-GI-NEXT:    orr w0, w8, w0, lsl #3
; CHECK-GI-NEXT:    ret
entry:
  %d = call i8 @llvm.fshl(i8 %a, i8 %b, i8 3)
  ret i8 %d
}

define i8 @fshr_i8_c(i8 %a, i8 %b) {
; CHECK-SD-LABEL: fshr_i8_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    lsl w8, w1, #24
; CHECK-SD-NEXT:    extr w0, w0, w8, #27
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_i8_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ubfx w8, w1, #3, #5
; CHECK-GI-NEXT:    orr w0, w8, w0, lsl #5
; CHECK-GI-NEXT:    ret
entry:
  %d = call i8 @llvm.fshr(i8 %a, i8 %b, i8 3)
  ret i8 %d
}

define i16 @fshl_i16_c(i16 %a, i16 %b) {
; CHECK-SD-LABEL: fshl_i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    lsl w8, w1, #16
; CHECK-SD-NEXT:    extr w0, w0, w8, #29
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ubfx w8, w1, #13, #3
; CHECK-GI-NEXT:    orr w0, w8, w0, lsl #3
; CHECK-GI-NEXT:    ret
entry:
  %d = call i16 @llvm.fshl(i16 %a, i16 %b, i16 3)
  ret i16 %d
}

define i16 @fshr_i16_c(i16 %a, i16 %b) {
; CHECK-SD-LABEL: fshr_i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    lsl w8, w1, #16
; CHECK-SD-NEXT:    extr w0, w0, w8, #19
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ubfx w8, w1, #3, #13
; CHECK-GI-NEXT:    orr w0, w8, w0, lsl #13
; CHECK-GI-NEXT:    ret
entry:
  %d = call i16 @llvm.fshr(i16 %a, i16 %b, i16 3)
  ret i16 %d
}

define i32 @fshl_i32_c(i32 %a, i32 %b) {
; CHECK-LABEL: fshl_i32_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    extr w0, w0, w1, #29
; CHECK-NEXT:    ret
entry:
  %d = call i32 @llvm.fshl(i32 %a, i32 %b, i32 3)
  ret i32 %d
}

define i32 @fshr_i32_c(i32 %a, i32 %b) {
; CHECK-LABEL: fshr_i32_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    extr w0, w0, w1, #3
; CHECK-NEXT:    ret
entry:
  %d = call i32 @llvm.fshr(i32 %a, i32 %b, i32 3)
  ret i32 %d
}

define i64 @fshl_i64_c(i64 %a, i64 %b) {
; CHECK-LABEL: fshl_i64_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    extr x0, x0, x1, #61
; CHECK-NEXT:    ret
entry:
  %d = call i64 @llvm.fshl(i64 %a, i64 %b, i64 3)
  ret i64 %d
}

define i64 @fshr_i64_c(i64 %a, i64 %b) {
; CHECK-LABEL: fshr_i64_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    extr x0, x0, x1, #3
; CHECK-NEXT:    ret
entry:
  %d = call i64 @llvm.fshr(i64 %a, i64 %b, i64 3)
  ret i64 %d
}

define i128 @fshl_i128_c(i128 %a, i128 %b) {
; CHECK-SD-LABEL: fshl_i128_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    extr x8, x0, x3, #61
; CHECK-SD-NEXT:    extr x1, x1, x0, #61
; CHECK-SD-NEXT:    mov x0, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_i128_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    lsr x8, x0, #61
; CHECK-GI-NEXT:    lsr x9, x3, #61
; CHECK-GI-NEXT:    orr x1, x8, x1, lsl #3
; CHECK-GI-NEXT:    orr x0, x9, x0, lsl #3
; CHECK-GI-NEXT:    ret
entry:
  %d = call i128 @llvm.fshl(i128 %a, i128 %b, i128 3)
  ret i128 %d
}

define i128 @fshr_i128_c(i128 %a, i128 %b) {
; CHECK-SD-LABEL: fshr_i128_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    extr x8, x3, x2, #3
; CHECK-SD-NEXT:    extr x1, x0, x3, #3
; CHECK-SD-NEXT:    mov x0, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_i128_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    lsl x8, x3, #61
; CHECK-GI-NEXT:    lsr x9, x3, #3
; CHECK-GI-NEXT:    orr x8, x8, x2, lsr #3
; CHECK-GI-NEXT:    orr x1, x9, x0, lsl #61
; CHECK-GI-NEXT:    mov x0, x8
; CHECK-GI-NEXT:    ret
entry:
  %d = call i128 @llvm.fshr(i128 %a, i128 %b, i128 3)
  ret i128 %d
}

define <8 x i8> @rotl_v8i8(<8 x i8> %a, <8 x i8> %c) {
; CHECK-LABEL: rotl_v8i8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v2.8b, #7
; CHECK-NEXT:    neg v3.8b, v1.8b
; CHECK-NEXT:    and v3.8b, v3.8b, v2.8b
; CHECK-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-NEXT:    neg v2.8b, v3.8b
; CHECK-NEXT:    ushl v1.8b, v0.8b, v1.8b
; CHECK-NEXT:    ushl v0.8b, v0.8b, v2.8b
; CHECK-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-NEXT:    ret
entry:
  %d = call <8 x i8> @llvm.fshl(<8 x i8> %a, <8 x i8> %a, <8 x i8> %c)
  ret <8 x i8> %d
}

define <8 x i8> @rotr_v8i8(<8 x i8> %a, <8 x i8> %c) {
; CHECK-SD-LABEL: rotr_v8i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v2.8b, #7
; CHECK-SD-NEXT:    neg v3.8b, v1.8b
; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    and v2.8b, v3.8b, v2.8b
; CHECK-SD-NEXT:    neg v1.8b, v1.8b
; CHECK-SD-NEXT:    ushl v2.8b, v0.8b, v2.8b
; CHECK-SD-NEXT:    ushl v0.8b, v0.8b, v1.8b
; CHECK-SD-NEXT:    orr v0.8b, v0.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v8i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v2.8b, #7
; CHECK-GI-NEXT:    neg v3.8b, v1.8b
; CHECK-GI-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v2.8b, v3.8b, v2.8b
; CHECK-GI-NEXT:    neg v1.8b, v1.8b
; CHECK-GI-NEXT:    ushl v1.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    ushl v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i8> @llvm.fshr(<8 x i8> %a, <8 x i8> %a, <8 x i8> %c)
  ret <8 x i8> %d
}

define <16 x i8> @rotl_v16i8(<16 x i8> %a, <16 x i8> %c) {
; CHECK-LABEL: rotl_v16i8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v2.16b, #7
; CHECK-NEXT:    neg v3.16b, v1.16b
; CHECK-NEXT:    and v3.16b, v3.16b, v2.16b
; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    neg v2.16b, v3.16b
; CHECK-NEXT:    ushl v1.16b, v0.16b, v1.16b
; CHECK-NEXT:    ushl v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-NEXT:    ret
entry:
  %d = call <16 x i8> @llvm.fshl(<16 x i8> %a, <16 x i8> %a, <16 x i8> %c)
  ret <16 x i8> %d
}

define <16 x i8> @rotr_v16i8(<16 x i8> %a, <16 x i8> %c) {
; CHECK-SD-LABEL: rotr_v16i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v2.16b, #7
; CHECK-SD-NEXT:    neg v3.16b, v1.16b
; CHECK-SD-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    and v2.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    neg v1.16b, v1.16b
; CHECK-SD-NEXT:    ushl v2.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    ushl v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v16i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v2.16b, #7
; CHECK-GI-NEXT:    neg v3.16b, v1.16b
; CHECK-GI-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    and v2.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    neg v1.16b, v1.16b
; CHECK-GI-NEXT:    ushl v1.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    ushl v0.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <16 x i8> @llvm.fshr(<16 x i8> %a, <16 x i8> %a, <16 x i8> %c)
  ret <16 x i8> %d
}

define <4 x i16> @rotl_v4i16(<4 x i16> %a, <4 x i16> %c) {
; CHECK-LABEL: rotl_v4i16:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v2.4h, #15
; CHECK-NEXT:    neg v3.4h, v1.4h
; CHECK-NEXT:    and v3.8b, v3.8b, v2.8b
; CHECK-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-NEXT:    neg v2.4h, v3.4h
; CHECK-NEXT:    ushl v1.4h, v0.4h, v1.4h
; CHECK-NEXT:    ushl v0.4h, v0.4h, v2.4h
; CHECK-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-NEXT:    ret
entry:
  %d = call <4 x i16> @llvm.fshl(<4 x i16> %a, <4 x i16> %a, <4 x i16> %c)
  ret <4 x i16> %d
}

define <4 x i16> @rotr_v4i16(<4 x i16> %a, <4 x i16> %c) {
; CHECK-SD-LABEL: rotr_v4i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v2.4h, #15
; CHECK-SD-NEXT:    neg v3.4h, v1.4h
; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    and v2.8b, v3.8b, v2.8b
; CHECK-SD-NEXT:    neg v1.4h, v1.4h
; CHECK-SD-NEXT:    ushl v2.4h, v0.4h, v2.4h
; CHECK-SD-NEXT:    ushl v0.4h, v0.4h, v1.4h
; CHECK-SD-NEXT:    orr v0.8b, v0.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v4i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v2.4h, #15
; CHECK-GI-NEXT:    neg v3.4h, v1.4h
; CHECK-GI-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v2.8b, v3.8b, v2.8b
; CHECK-GI-NEXT:    neg v1.4h, v1.4h
; CHECK-GI-NEXT:    ushl v1.4h, v0.4h, v1.4h
; CHECK-GI-NEXT:    ushl v0.4h, v0.4h, v2.4h
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i16> @llvm.fshr(<4 x i16> %a, <4 x i16> %a, <4 x i16> %c)
  ret <4 x i16> %d
}

define <7 x i16> @rotl_v7i16(<7 x i16> %a, <7 x i16> %c) {
; CHECK-SD-LABEL: rotl_v7i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v2.8h, #15
; CHECK-SD-NEXT:    neg v3.8h, v1.8h
; CHECK-SD-NEXT:    and v3.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    neg v2.8h, v3.8h
; CHECK-SD-NEXT:    ushl v1.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v7i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #0 // =0x0
; CHECK-GI-NEXT:    mov w9, #15 // =0xf
; CHECK-GI-NEXT:    fmov s2, w8
; CHECK-GI-NEXT:    fmov s3, w9
; CHECK-GI-NEXT:    mov v2.h[1], w8
; CHECK-GI-NEXT:    mov v3.h[1], w9
; CHECK-GI-NEXT:    mov v2.h[2], w8
; CHECK-GI-NEXT:    mov v3.h[2], w9
; CHECK-GI-NEXT:    mov v2.h[3], w8
; CHECK-GI-NEXT:    mov v3.h[3], w9
; CHECK-GI-NEXT:    mov v2.h[4], w8
; CHECK-GI-NEXT:    mov v3.h[4], w9
; CHECK-GI-NEXT:    mov v2.h[5], w8
; CHECK-GI-NEXT:    mov v3.h[5], w9
; CHECK-GI-NEXT:    mov v2.h[6], w8
; CHECK-GI-NEXT:    mov v3.h[6], w9
; CHECK-GI-NEXT:    sub v2.8h, v2.8h, v1.8h
; CHECK-GI-NEXT:    and v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    and v2.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    ushl v1.8h, v0.8h, v1.8h
; CHECK-GI-NEXT:    neg v2.8h, v2.8h
; CHECK-GI-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i16> @llvm.fshl(<7 x i16> %a, <7 x i16> %a, <7 x i16> %c)
  ret <7 x i16> %d
}

define <7 x i16> @rotr_v7i16(<7 x i16> %a, <7 x i16> %c) {
; CHECK-SD-LABEL: rotr_v7i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v2.8h, #15
; CHECK-SD-NEXT:    neg v3.8h, v1.8h
; CHECK-SD-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    and v2.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    neg v1.8h, v1.8h
; CHECK-SD-NEXT:    ushl v2.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    ushl v0.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v7i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #0 // =0x0
; CHECK-GI-NEXT:    mov w9, #15 // =0xf
; CHECK-GI-NEXT:    fmov s2, w8
; CHECK-GI-NEXT:    fmov s3, w9
; CHECK-GI-NEXT:    mov v2.h[1], w8
; CHECK-GI-NEXT:    mov v3.h[1], w9
; CHECK-GI-NEXT:    mov v2.h[2], w8
; CHECK-GI-NEXT:    mov v3.h[2], w9
; CHECK-GI-NEXT:    mov v2.h[3], w8
; CHECK-GI-NEXT:    mov v3.h[3], w9
; CHECK-GI-NEXT:    mov v2.h[4], w8
; CHECK-GI-NEXT:    mov v3.h[4], w9
; CHECK-GI-NEXT:    mov v2.h[5], w8
; CHECK-GI-NEXT:    mov v3.h[5], w9
; CHECK-GI-NEXT:    mov v2.h[6], w8
; CHECK-GI-NEXT:    mov v3.h[6], w9
; CHECK-GI-NEXT:    sub v2.8h, v2.8h, v1.8h
; CHECK-GI-NEXT:    and v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    neg v1.8h, v1.8h
; CHECK-GI-NEXT:    and v2.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    ushl v1.8h, v0.8h, v1.8h
; CHECK-GI-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i16> @llvm.fshr(<7 x i16> %a, <7 x i16> %a, <7 x i16> %c)
  ret <7 x i16> %d
}

define <8 x i16> @rotl_v8i16(<8 x i16> %a, <8 x i16> %c) {
; CHECK-LABEL: rotl_v8i16:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v2.8h, #15
; CHECK-NEXT:    neg v3.8h, v1.8h
; CHECK-NEXT:    and v3.16b, v3.16b, v2.16b
; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    neg v2.8h, v3.8h
; CHECK-NEXT:    ushl v1.8h, v0.8h, v1.8h
; CHECK-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-NEXT:    ret
entry:
  %d = call <8 x i16> @llvm.fshl(<8 x i16> %a, <8 x i16> %a, <8 x i16> %c)
  ret <8 x i16> %d
}

define <8 x i16> @rotr_v8i16(<8 x i16> %a, <8 x i16> %c) {
; CHECK-SD-LABEL: rotr_v8i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v2.8h, #15
; CHECK-SD-NEXT:    neg v3.8h, v1.8h
; CHECK-SD-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    and v2.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    neg v1.8h, v1.8h
; CHECK-SD-NEXT:    ushl v2.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    ushl v0.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v8i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v2.8h, #15
; CHECK-GI-NEXT:    neg v3.8h, v1.8h
; CHECK-GI-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    and v2.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    neg v1.8h, v1.8h
; CHECK-GI-NEXT:    ushl v1.8h, v0.8h, v1.8h
; CHECK-GI-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i16> @llvm.fshr(<8 x i16> %a, <8 x i16> %a, <8 x i16> %c)
  ret <8 x i16> %d
}

define <16 x i16> @rotl_v16i16(<16 x i16> %a, <16 x i16> %c) {
; CHECK-LABEL: rotl_v16i16:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v4.8h, #15
; CHECK-NEXT:    neg v5.8h, v2.8h
; CHECK-NEXT:    neg v6.8h, v3.8h
; CHECK-NEXT:    and v5.16b, v5.16b, v4.16b
; CHECK-NEXT:    and v6.16b, v6.16b, v4.16b
; CHECK-NEXT:    and v2.16b, v2.16b, v4.16b
; CHECK-NEXT:    and v3.16b, v3.16b, v4.16b
; CHECK-NEXT:    neg v4.8h, v5.8h
; CHECK-NEXT:    neg v5.8h, v6.8h
; CHECK-NEXT:    ushl v2.8h, v0.8h, v2.8h
; CHECK-NEXT:    ushl v3.8h, v1.8h, v3.8h
; CHECK-NEXT:    ushl v0.8h, v0.8h, v4.8h
; CHECK-NEXT:    ushl v1.8h, v1.8h, v5.8h
; CHECK-NEXT:    orr v0.16b, v2.16b, v0.16b
; CHECK-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-NEXT:    ret
entry:
  %d = call <16 x i16> @llvm.fshl(<16 x i16> %a, <16 x i16> %a, <16 x i16> %c)
  ret <16 x i16> %d
}

define <16 x i16> @rotr_v16i16(<16 x i16> %a, <16 x i16> %c) {
; CHECK-SD-LABEL: rotr_v16i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v4.8h, #15
; CHECK-SD-NEXT:    neg v5.8h, v2.8h
; CHECK-SD-NEXT:    neg v6.8h, v3.8h
; CHECK-SD-NEXT:    and v2.16b, v2.16b, v4.16b
; CHECK-SD-NEXT:    and v3.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    and v5.16b, v5.16b, v4.16b
; CHECK-SD-NEXT:    and v4.16b, v6.16b, v4.16b
; CHECK-SD-NEXT:    neg v2.8h, v2.8h
; CHECK-SD-NEXT:    neg v3.8h, v3.8h
; CHECK-SD-NEXT:    ushl v5.8h, v0.8h, v5.8h
; CHECK-SD-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    ushl v2.8h, v1.8h, v4.8h
; CHECK-SD-NEXT:    ushl v1.8h, v1.8h, v3.8h
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v5.16b
; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v16i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v4.8h, #15
; CHECK-GI-NEXT:    neg v6.8h, v3.8h
; CHECK-GI-NEXT:    and v5.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    neg v2.8h, v2.8h
; CHECK-GI-NEXT:    and v3.16b, v3.16b, v4.16b
; CHECK-GI-NEXT:    neg v5.8h, v5.8h
; CHECK-GI-NEXT:    and v2.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    and v4.16b, v6.16b, v4.16b
; CHECK-GI-NEXT:    neg v3.8h, v3.8h
; CHECK-GI-NEXT:    ushl v5.8h, v0.8h, v5.8h
; CHECK-GI-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    ushl v3.8h, v1.8h, v3.8h
; CHECK-GI-NEXT:    ushl v1.8h, v1.8h, v4.8h
; CHECK-GI-NEXT:    orr v0.16b, v5.16b, v0.16b
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <16 x i16> @llvm.fshr(<16 x i16> %a, <16 x i16> %a, <16 x i16> %c)
  ret <16 x i16> %d
}

define <2 x i32> @rotl_v2i32(<2 x i32> %a, <2 x i32> %c) {
; CHECK-LABEL: rotl_v2i32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v2.2s, #31
; CHECK-NEXT:    neg v3.2s, v1.2s
; CHECK-NEXT:    and v3.8b, v3.8b, v2.8b
; CHECK-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-NEXT:    neg v2.2s, v3.2s
; CHECK-NEXT:    ushl v1.2s, v0.2s, v1.2s
; CHECK-NEXT:    ushl v0.2s, v0.2s, v2.2s
; CHECK-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-NEXT:    ret
entry:
  %d = call <2 x i32> @llvm.fshl(<2 x i32> %a, <2 x i32> %a, <2 x i32> %c)
  ret <2 x i32> %d
}

define <2 x i32> @rotr_v2i32(<2 x i32> %a, <2 x i32> %c) {
; CHECK-SD-LABEL: rotr_v2i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v2.2s, #31
; CHECK-SD-NEXT:    neg v3.2s, v1.2s
; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-SD-NEXT:    and v2.8b, v3.8b, v2.8b
; CHECK-SD-NEXT:    neg v1.2s, v1.2s
; CHECK-SD-NEXT:    ushl v2.2s, v0.2s, v2.2s
; CHECK-SD-NEXT:    ushl v0.2s, v0.2s, v1.2s
; CHECK-SD-NEXT:    orr v0.8b, v0.8b, v2.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v2i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v2.2s, #31
; CHECK-GI-NEXT:    neg v3.2s, v1.2s
; CHECK-GI-NEXT:    and v1.8b, v1.8b, v2.8b
; CHECK-GI-NEXT:    and v2.8b, v3.8b, v2.8b
; CHECK-GI-NEXT:    neg v1.2s, v1.2s
; CHECK-GI-NEXT:    ushl v1.2s, v0.2s, v1.2s
; CHECK-GI-NEXT:    ushl v0.2s, v0.2s, v2.2s
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i32> @llvm.fshr(<2 x i32> %a, <2 x i32> %a, <2 x i32> %c)
  ret <2 x i32> %d
}

define <4 x i32> @rotl_v4i32(<4 x i32> %a, <4 x i32> %c) {
; CHECK-LABEL: rotl_v4i32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v2.4s, #31
; CHECK-NEXT:    neg v3.4s, v1.4s
; CHECK-NEXT:    and v3.16b, v3.16b, v2.16b
; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-NEXT:    neg v2.4s, v3.4s
; CHECK-NEXT:    ushl v1.4s, v0.4s, v1.4s
; CHECK-NEXT:    ushl v0.4s, v0.4s, v2.4s
; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-NEXT:    ret
entry:
  %d = call <4 x i32> @llvm.fshl(<4 x i32> %a, <4 x i32> %a, <4 x i32> %c)
  ret <4 x i32> %d
}

define <4 x i32> @rotr_v4i32(<4 x i32> %a, <4 x i32> %c) {
; CHECK-SD-LABEL: rotr_v4i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v2.4s, #31
; CHECK-SD-NEXT:    neg v3.4s, v1.4s
; CHECK-SD-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    and v2.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    neg v1.4s, v1.4s
; CHECK-SD-NEXT:    ushl v2.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    ushl v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v4i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v2.4s, #31
; CHECK-GI-NEXT:    neg v3.4s, v1.4s
; CHECK-GI-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    and v2.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    neg v1.4s, v1.4s
; CHECK-GI-NEXT:    ushl v1.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v2.4s
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i32> @llvm.fshr(<4 x i32> %a, <4 x i32> %a, <4 x i32> %c)
  ret <4 x i32> %d
}

define <7 x i32> @rotl_v7i32(<7 x i32> %a, <7 x i32> %c) {
; CHECK-SD-LABEL: rotl_v7i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    fmov s0, w7
; CHECK-SD-NEXT:    mov x8, sp
; CHECK-SD-NEXT:    ldr s1, [sp, #24]
; CHECK-SD-NEXT:    fmov s2, w0
; CHECK-SD-NEXT:    add x9, sp, #40
; CHECK-SD-NEXT:    fmov s5, w4
; CHECK-SD-NEXT:    movi v3.4s, #31
; CHECK-SD-NEXT:    ld1 { v0.s }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #8
; CHECK-SD-NEXT:    mov v2.s[1], w1
; CHECK-SD-NEXT:    mov v5.s[1], w5
; CHECK-SD-NEXT:    ld1 { v0.s }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #32
; CHECK-SD-NEXT:    ld1 { v1.s }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #16
; CHECK-SD-NEXT:    mov v2.s[2], w2
; CHECK-SD-NEXT:    mov v5.s[2], w6
; CHECK-SD-NEXT:    ld1 { v0.s }[3], [x8]
; CHECK-SD-NEXT:    ld1 { v1.s }[2], [x9]
; CHECK-SD-NEXT:    neg v4.4s, v0.4s
; CHECK-SD-NEXT:    mov v2.s[3], w3
; CHECK-SD-NEXT:    and v0.16b, v0.16b, v3.16b
; CHECK-SD-NEXT:    neg v6.4s, v1.4s
; CHECK-SD-NEXT:    and v1.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    and v4.16b, v4.16b, v3.16b
; CHECK-SD-NEXT:    and v6.16b, v6.16b, v3.16b
; CHECK-SD-NEXT:    ushl v0.4s, v2.4s, v0.4s
; CHECK-SD-NEXT:    ushl v1.4s, v5.4s, v1.4s
; CHECK-SD-NEXT:    neg v4.4s, v4.4s
; CHECK-SD-NEXT:    neg v3.4s, v6.4s
; CHECK-SD-NEXT:    ushl v2.4s, v2.4s, v4.4s
; CHECK-SD-NEXT:    ushl v3.4s, v5.4s, v3.4s
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    mov w1, v0.s[1]
; CHECK-SD-NEXT:    mov w2, v0.s[2]
; CHECK-SD-NEXT:    mov w3, v0.s[3]
; CHECK-SD-NEXT:    mov w5, v1.s[1]
; CHECK-SD-NEXT:    mov w6, v1.s[2]
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    fmov w4, s1
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v7i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov v1.s[0], w7
; CHECK-GI-NEXT:    ldr s0, [sp, #24]
; CHECK-GI-NEXT:    mov v2.s[0], wzr
; CHECK-GI-NEXT:    mov v3.s[0], w7
; CHECK-GI-NEXT:    mov x9, sp
; CHECK-GI-NEXT:    ldr s5, [sp, #32]
; CHECK-GI-NEXT:    mov v4.16b, v0.16b
; CHECK-GI-NEXT:    mov w8, #31 // =0x1f
; CHECK-GI-NEXT:    ldr s7, [sp]
; CHECK-GI-NEXT:    mov v16.s[0], w8
; CHECK-GI-NEXT:    mov v6.s[0], w0
; CHECK-GI-NEXT:    ldr s18, [sp, #40]
; CHECK-GI-NEXT:    ld1 { v1.s }[1], [x9]
; CHECK-GI-NEXT:    mov v2.s[1], wzr
; CHECK-GI-NEXT:    add x9, sp, #8
; CHECK-GI-NEXT:    mov v4.s[1], v5.s[0]
; CHECK-GI-NEXT:    mov v3.s[1], v7.s[0]
; CHECK-GI-NEXT:    mov v7.s[0], w0
; CHECK-GI-NEXT:    mov v19.s[0], w8
; CHECK-GI-NEXT:    ldr s17, [sp, #8]
; CHECK-GI-NEXT:    mov v20.s[0], w4
; CHECK-GI-NEXT:    ld1 { v1.s }[2], [x9]
; CHECK-GI-NEXT:    mov v16.s[1], w8
; CHECK-GI-NEXT:    add x9, sp, #16
; CHECK-GI-NEXT:    mov v2.s[2], wzr
; CHECK-GI-NEXT:    mov v6.s[1], w1
; CHECK-GI-NEXT:    mov v21.s[0], w4
; CHECK-GI-NEXT:    mov v4.s[2], v18.s[0]
; CHECK-GI-NEXT:    mov v7.s[1], w1
; CHECK-GI-NEXT:    mov v3.s[2], v17.s[0]
; CHECK-GI-NEXT:    ld1 { v1.s }[3], [x9]
; CHECK-GI-NEXT:    mov v0.s[1], v5.s[0]
; CHECK-GI-NEXT:    mov v19.s[1], w8
; CHECK-GI-NEXT:    movi v5.4s, #31
; CHECK-GI-NEXT:    mov v16.s[2], w8
; CHECK-GI-NEXT:    ldr s17, [sp, #16]
; CHECK-GI-NEXT:    mov v6.s[2], w2
; CHECK-GI-NEXT:    mov v20.s[1], w5
; CHECK-GI-NEXT:    mov v21.s[1], w5
; CHECK-GI-NEXT:    sub v2.4s, v2.4s, v4.4s
; CHECK-GI-NEXT:    neg v1.4s, v1.4s
; CHECK-GI-NEXT:    mov v7.s[2], w2
; CHECK-GI-NEXT:    mov v3.s[3], v17.s[0]
; CHECK-GI-NEXT:    mov v0.s[2], v18.s[0]
; CHECK-GI-NEXT:    mov v19.s[2], w8
; CHECK-GI-NEXT:    and v1.16b, v1.16b, v5.16b
; CHECK-GI-NEXT:    and v2.16b, v2.16b, v16.16b
; CHECK-GI-NEXT:    mov v6.s[3], w3
; CHECK-GI-NEXT:    mov v7.s[3], w3
; CHECK-GI-NEXT:    mov v20.s[2], w6
; CHECK-GI-NEXT:    mov v21.s[2], w6
; CHECK-GI-NEXT:    and v3.16b, v3.16b, v5.16b
; CHECK-GI-NEXT:    and v0.16b, v0.16b, v19.16b
; CHECK-GI-NEXT:    neg v1.4s, v1.4s
; CHECK-GI-NEXT:    neg v2.4s, v2.4s
; CHECK-GI-NEXT:    ushl v3.4s, v6.4s, v3.4s
; CHECK-GI-NEXT:    ushl v0.4s, v20.4s, v0.4s
; CHECK-GI-NEXT:    ushl v1.4s, v7.4s, v1.4s
; CHECK-GI-NEXT:    ushl v2.4s, v21.4s, v2.4s
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    mov s2, v1.s[1]
; CHECK-GI-NEXT:    mov s3, v1.s[2]
; CHECK-GI-NEXT:    mov s4, v1.s[3]
; CHECK-GI-NEXT:    mov s5, v0.s[1]
; CHECK-GI-NEXT:    mov s6, v0.s[2]
; CHECK-GI-NEXT:    fmov w0, s1
; CHECK-GI-NEXT:    fmov w4, s0
; CHECK-GI-NEXT:    fmov w1, s2
; CHECK-GI-NEXT:    fmov w2, s3
; CHECK-GI-NEXT:    fmov w3, s4
; CHECK-GI-NEXT:    fmov w5, s5
; CHECK-GI-NEXT:    fmov w6, s6
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i32> @llvm.fshl(<7 x i32> %a, <7 x i32> %a, <7 x i32> %c)
  ret <7 x i32> %d
}

define <7 x i32> @rotr_v7i32(<7 x i32> %a, <7 x i32> %c) {
; CHECK-SD-LABEL: rotr_v7i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    fmov s0, w7
; CHECK-SD-NEXT:    mov x8, sp
; CHECK-SD-NEXT:    fmov s2, w0
; CHECK-SD-NEXT:    ldr s1, [sp, #24]
; CHECK-SD-NEXT:    add x9, sp, #32
; CHECK-SD-NEXT:    fmov s4, w4
; CHECK-SD-NEXT:    movi v3.4s, #31
; CHECK-SD-NEXT:    ld1 { v0.s }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #8
; CHECK-SD-NEXT:    mov v2.s[1], w1
; CHECK-SD-NEXT:    ld1 { v1.s }[1], [x9]
; CHECK-SD-NEXT:    add x9, sp, #40
; CHECK-SD-NEXT:    mov v4.s[1], w5
; CHECK-SD-NEXT:    ld1 { v0.s }[2], [x8]
; CHECK-SD-NEXT:    add x8, sp, #16
; CHECK-SD-NEXT:    ld1 { v1.s }[2], [x9]
; CHECK-SD-NEXT:    mov v2.s[2], w2
; CHECK-SD-NEXT:    mov v4.s[2], w6
; CHECK-SD-NEXT:    ld1 { v0.s }[3], [x8]
; CHECK-SD-NEXT:    neg v6.4s, v1.4s
; CHECK-SD-NEXT:    and v1.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    mov v2.s[3], w3
; CHECK-SD-NEXT:    neg v5.4s, v0.4s
; CHECK-SD-NEXT:    and v0.16b, v0.16b, v3.16b
; CHECK-SD-NEXT:    neg v1.4s, v1.4s
; CHECK-SD-NEXT:    and v5.16b, v5.16b, v3.16b
; CHECK-SD-NEXT:    neg v0.4s, v0.4s
; CHECK-SD-NEXT:    and v3.16b, v6.16b, v3.16b
; CHECK-SD-NEXT:    ushl v1.4s, v4.4s, v1.4s
; CHECK-SD-NEXT:    ushl v5.4s, v2.4s, v5.4s
; CHECK-SD-NEXT:    ushl v0.4s, v2.4s, v0.4s
; CHECK-SD-NEXT:    ushl v2.4s, v4.4s, v3.4s
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v5.16b
; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    mov w1, v0.s[1]
; CHECK-SD-NEXT:    mov w2, v0.s[2]
; CHECK-SD-NEXT:    mov w3, v0.s[3]
; CHECK-SD-NEXT:    mov w5, v1.s[1]
; CHECK-SD-NEXT:    mov w6, v1.s[2]
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    fmov w4, s1
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v7i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov v0.s[0], w7
; CHECK-GI-NEXT:    ldr s2, [sp]
; CHECK-GI-NEXT:    mov v1.s[0], w7
; CHECK-GI-NEXT:    mov w8, #31 // =0x1f
; CHECK-GI-NEXT:    ldr s7, [sp, #8]
; CHECK-GI-NEXT:    mov v3.s[0], wzr
; CHECK-GI-NEXT:    mov v5.s[0], w8
; CHECK-GI-NEXT:    mov x11, sp
; CHECK-GI-NEXT:    ldr s16, [sp, #32]
; CHECK-GI-NEXT:    mov v4.s[0], w0
; CHECK-GI-NEXT:    mov v17.s[0], w0
; CHECK-GI-NEXT:    ldr s6, [sp, #16]
; CHECK-GI-NEXT:    mov v0.s[1], v2.s[0]
; CHECK-GI-NEXT:    ldr s2, [sp, #24]
; CHECK-GI-NEXT:    ld1 { v1.s }[1], [x11]
; CHECK-GI-NEXT:    mov v3.s[1], wzr
; CHECK-GI-NEXT:    add x10, sp, #8
; CHECK-GI-NEXT:    ldr s18, [sp, #40]
; CHECK-GI-NEXT:    mov v19.16b, v2.16b
; CHECK-GI-NEXT:    mov v2.s[1], v16.s[0]
; CHECK-GI-NEXT:    mov v5.s[1], w8
; CHECK-GI-NEXT:    ld1 { v1.s }[2], [x10]
; CHECK-GI-NEXT:    mov v4.s[1], w1
; CHECK-GI-NEXT:    mov v17.s[1], w1
; CHECK-GI-NEXT:    mov v0.s[2], v7.s[0]
; CHECK-GI-NEXT:    mov v7.s[0], w8
; CHECK-GI-NEXT:    mov v20.s[0], w4
; CHECK-GI-NEXT:    mov v19.s[1], v16.s[0]
; CHECK-GI-NEXT:    add x9, sp, #16
; CHECK-GI-NEXT:    movi v16.4s, #31
; CHECK-GI-NEXT:    mov v2.s[2], v18.s[0]
; CHECK-GI-NEXT:    mov v3.s[2], wzr
; CHECK-GI-NEXT:    mov v5.s[2], w8
; CHECK-GI-NEXT:    ld1 { v1.s }[3], [x9]
; CHECK-GI-NEXT:    mov v4.s[2], w2
; CHECK-GI-NEXT:    mov v17.s[2], w2
; CHECK-GI-NEXT:    mov v0.s[3], v6.s[0]
; CHECK-GI-NEXT:    mov v6.s[0], w4
; CHECK-GI-NEXT:    mov v7.s[1], w8
; CHECK-GI-NEXT:    mov v19.s[2], v18.s[0]
; CHECK-GI-NEXT:    mov v20.s[1], w5
; CHECK-GI-NEXT:    neg v1.4s, v1.4s
; CHECK-GI-NEXT:    and v2.16b, v2.16b, v5.16b
; CHECK-GI-NEXT:    mov v4.s[3], w3
; CHECK-GI-NEXT:    mov v17.s[3], w3
; CHECK-GI-NEXT:    mov v6.s[1], w5
; CHECK-GI-NEXT:    mov v7.s[2], w8
; CHECK-GI-NEXT:    and v0.16b, v0.16b, v16.16b
; CHECK-GI-NEXT:    sub v3.4s, v3.4s, v19.4s
; CHECK-GI-NEXT:    mov v20.s[2], w6
; CHECK-GI-NEXT:    and v1.16b, v1.16b, v16.16b
; CHECK-GI-NEXT:    neg v2.4s, v2.4s
; CHECK-GI-NEXT:    neg v0.4s, v0.4s
; CHECK-GI-NEXT:    mov v6.s[2], w6
; CHECK-GI-NEXT:    and v3.16b, v3.16b, v7.16b
; CHECK-GI-NEXT:    ushl v1.4s, v17.4s, v1.4s
; CHECK-GI-NEXT:    ushl v2.4s, v20.4s, v2.4s
; CHECK-GI-NEXT:    ushl v0.4s, v4.4s, v0.4s
; CHECK-GI-NEXT:    ushl v3.4s, v6.4s, v3.4s
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    orr v1.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    mov s2, v0.s[1]
; CHECK-GI-NEXT:    mov s3, v0.s[2]
; CHECK-GI-NEXT:    mov s4, v0.s[3]
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    mov s5, v1.s[1]
; CHECK-GI-NEXT:    mov s6, v1.s[2]
; CHECK-GI-NEXT:    fmov w4, s1
; CHECK-GI-NEXT:    fmov w1, s2
; CHECK-GI-NEXT:    fmov w2, s3
; CHECK-GI-NEXT:    fmov w3, s4
; CHECK-GI-NEXT:    fmov w5, s5
; CHECK-GI-NEXT:    fmov w6, s6
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i32> @llvm.fshr(<7 x i32> %a, <7 x i32> %a, <7 x i32> %c)
  ret <7 x i32> %d
}

define <8 x i32> @rotl_v8i32(<8 x i32> %a, <8 x i32> %c) {
; CHECK-LABEL: rotl_v8i32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v4.4s, #31
; CHECK-NEXT:    neg v5.4s, v2.4s
; CHECK-NEXT:    neg v6.4s, v3.4s
; CHECK-NEXT:    and v5.16b, v5.16b, v4.16b
; CHECK-NEXT:    and v6.16b, v6.16b, v4.16b
; CHECK-NEXT:    and v2.16b, v2.16b, v4.16b
; CHECK-NEXT:    and v3.16b, v3.16b, v4.16b
; CHECK-NEXT:    neg v4.4s, v5.4s
; CHECK-NEXT:    neg v5.4s, v6.4s
; CHECK-NEXT:    ushl v2.4s, v0.4s, v2.4s
; CHECK-NEXT:    ushl v3.4s, v1.4s, v3.4s
; CHECK-NEXT:    ushl v0.4s, v0.4s, v4.4s
; CHECK-NEXT:    ushl v1.4s, v1.4s, v5.4s
; CHECK-NEXT:    orr v0.16b, v2.16b, v0.16b
; CHECK-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-NEXT:    ret
entry:
  %d = call <8 x i32> @llvm.fshl(<8 x i32> %a, <8 x i32> %a, <8 x i32> %c)
  ret <8 x i32> %d
}

define <8 x i32> @rotr_v8i32(<8 x i32> %a, <8 x i32> %c) {
; CHECK-SD-LABEL: rotr_v8i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v4.4s, #31
; CHECK-SD-NEXT:    neg v5.4s, v2.4s
; CHECK-SD-NEXT:    neg v6.4s, v3.4s
; CHECK-SD-NEXT:    and v2.16b, v2.16b, v4.16b
; CHECK-SD-NEXT:    and v3.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    and v5.16b, v5.16b, v4.16b
; CHECK-SD-NEXT:    and v4.16b, v6.16b, v4.16b
; CHECK-SD-NEXT:    neg v2.4s, v2.4s
; CHECK-SD-NEXT:    neg v3.4s, v3.4s
; CHECK-SD-NEXT:    ushl v5.4s, v0.4s, v5.4s
; CHECK-SD-NEXT:    ushl v0.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    ushl v2.4s, v1.4s, v4.4s
; CHECK-SD-NEXT:    ushl v1.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v5.16b
; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v8i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v4.4s, #31
; CHECK-GI-NEXT:    neg v6.4s, v3.4s
; CHECK-GI-NEXT:    and v5.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    neg v2.4s, v2.4s
; CHECK-GI-NEXT:    and v3.16b, v3.16b, v4.16b
; CHECK-GI-NEXT:    neg v5.4s, v5.4s
; CHECK-GI-NEXT:    and v2.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    and v4.16b, v6.16b, v4.16b
; CHECK-GI-NEXT:    neg v3.4s, v3.4s
; CHECK-GI-NEXT:    ushl v5.4s, v0.4s, v5.4s
; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v2.4s
; CHECK-GI-NEXT:    ushl v3.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v4.4s
; CHECK-GI-NEXT:    orr v0.16b, v5.16b, v0.16b
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i32> @llvm.fshr(<8 x i32> %a, <8 x i32> %a, <8 x i32> %c)
  ret <8 x i32> %d
}

define <2 x i64> @rotl_v2i64(<2 x i64> %a, <2 x i64> %c) {
; CHECK-SD-LABEL: rotl_v2i64:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mov w8, #63 // =0x3f
; CHECK-SD-NEXT:    neg v3.2d, v1.2d
; CHECK-SD-NEXT:    dup v2.2d, x8
; CHECK-SD-NEXT:    and v3.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    neg v2.2d, v3.2d
; CHECK-SD-NEXT:    ushl v1.2d, v0.2d, v1.2d
; CHECK-SD-NEXT:    ushl v0.2d, v0.2d, v2.2d
; CHECK-SD-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v2i64:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    adrp x8, .LCPI60_0
; CHECK-GI-NEXT:    neg v2.2d, v1.2d
; CHECK-GI-NEXT:    ldr q3, [x8, :lo12:.LCPI60_0]
; CHECK-GI-NEXT:    and v2.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    and v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    neg v2.2d, v2.2d
; CHECK-GI-NEXT:    ushl v1.2d, v0.2d, v1.2d
; CHECK-GI-NEXT:    ushl v0.2d, v0.2d, v2.2d
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i64> @llvm.fshl(<2 x i64> %a, <2 x i64> %a, <2 x i64> %c)
  ret <2 x i64> %d
}

define <2 x i64> @rotr_v2i64(<2 x i64> %a, <2 x i64> %c) {
; CHECK-SD-LABEL: rotr_v2i64:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mov w8, #63 // =0x3f
; CHECK-SD-NEXT:    neg v3.2d, v1.2d
; CHECK-SD-NEXT:    dup v2.2d, x8
; CHECK-SD-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    and v2.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    neg v1.2d, v1.2d
; CHECK-SD-NEXT:    ushl v2.2d, v0.2d, v2.2d
; CHECK-SD-NEXT:    ushl v0.2d, v0.2d, v1.2d
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v2i64:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    adrp x8, .LCPI61_0
; CHECK-GI-NEXT:    neg v3.2d, v1.2d
; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI61_0]
; CHECK-GI-NEXT:    and v1.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    and v2.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    neg v1.2d, v1.2d
; CHECK-GI-NEXT:    ushl v1.2d, v0.2d, v1.2d
; CHECK-GI-NEXT:    ushl v0.2d, v0.2d, v2.2d
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i64> @llvm.fshr(<2 x i64> %a, <2 x i64> %a, <2 x i64> %c)
  ret <2 x i64> %d
}

define <4 x i64> @rotl_v4i64(<4 x i64> %a, <4 x i64> %c) {
; CHECK-SD-LABEL: rotl_v4i64:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mov w8, #63 // =0x3f
; CHECK-SD-NEXT:    neg v5.2d, v2.2d
; CHECK-SD-NEXT:    neg v6.2d, v3.2d
; CHECK-SD-NEXT:    dup v4.2d, x8
; CHECK-SD-NEXT:    and v5.16b, v5.16b, v4.16b
; CHECK-SD-NEXT:    and v6.16b, v6.16b, v4.16b
; CHECK-SD-NEXT:    and v2.16b, v2.16b, v4.16b
; CHECK-SD-NEXT:    and v3.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    neg v4.2d, v5.2d
; CHECK-SD-NEXT:    neg v5.2d, v6.2d
; CHECK-SD-NEXT:    ushl v2.2d, v0.2d, v2.2d
; CHECK-SD-NEXT:    ushl v3.2d, v1.2d, v3.2d
; CHECK-SD-NEXT:    ushl v0.2d, v0.2d, v4.2d
; CHECK-SD-NEXT:    ushl v1.2d, v1.2d, v5.2d
; CHECK-SD-NEXT:    orr v0.16b, v2.16b, v0.16b
; CHECK-SD-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v4i64:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    adrp x8, .LCPI62_0
; CHECK-GI-NEXT:    neg v4.2d, v2.2d
; CHECK-GI-NEXT:    neg v5.2d, v3.2d
; CHECK-GI-NEXT:    ldr q6, [x8, :lo12:.LCPI62_0]
; CHECK-GI-NEXT:    and v4.16b, v4.16b, v6.16b
; CHECK-GI-NEXT:    and v5.16b, v5.16b, v6.16b
; CHECK-GI-NEXT:    and v2.16b, v2.16b, v6.16b
; CHECK-GI-NEXT:    and v3.16b, v3.16b, v6.16b
; CHECK-GI-NEXT:    neg v4.2d, v4.2d
; CHECK-GI-NEXT:    neg v5.2d, v5.2d
; CHECK-GI-NEXT:    ushl v2.2d, v0.2d, v2.2d
; CHECK-GI-NEXT:    ushl v3.2d, v1.2d, v3.2d
; CHECK-GI-NEXT:    ushl v0.2d, v0.2d, v4.2d
; CHECK-GI-NEXT:    ushl v1.2d, v1.2d, v5.2d
; CHECK-GI-NEXT:    orr v0.16b, v2.16b, v0.16b
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i64> @llvm.fshl(<4 x i64> %a, <4 x i64> %a, <4 x i64> %c)
  ret <4 x i64> %d
}

define <4 x i64> @rotr_v4i64(<4 x i64> %a, <4 x i64> %c) {
; CHECK-SD-LABEL: rotr_v4i64:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mov w8, #63 // =0x3f
; CHECK-SD-NEXT:    neg v5.2d, v2.2d
; CHECK-SD-NEXT:    neg v6.2d, v3.2d
; CHECK-SD-NEXT:    dup v4.2d, x8
; CHECK-SD-NEXT:    and v2.16b, v2.16b, v4.16b
; CHECK-SD-NEXT:    and v3.16b, v3.16b, v4.16b
; CHECK-SD-NEXT:    and v5.16b, v5.16b, v4.16b
; CHECK-SD-NEXT:    and v4.16b, v6.16b, v4.16b
; CHECK-SD-NEXT:    neg v2.2d, v2.2d
; CHECK-SD-NEXT:    neg v3.2d, v3.2d
; CHECK-SD-NEXT:    ushl v5.2d, v0.2d, v5.2d
; CHECK-SD-NEXT:    ushl v0.2d, v0.2d, v2.2d
; CHECK-SD-NEXT:    ushl v2.2d, v1.2d, v4.2d
; CHECK-SD-NEXT:    ushl v1.2d, v1.2d, v3.2d
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v5.16b
; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v4i64:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    adrp x8, .LCPI63_0
; CHECK-GI-NEXT:    neg v6.2d, v3.2d
; CHECK-GI-NEXT:    ldr q4, [x8, :lo12:.LCPI63_0]
; CHECK-GI-NEXT:    and v5.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    neg v2.2d, v2.2d
; CHECK-GI-NEXT:    and v3.16b, v3.16b, v4.16b
; CHECK-GI-NEXT:    neg v5.2d, v5.2d
; CHECK-GI-NEXT:    and v2.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    and v4.16b, v6.16b, v4.16b
; CHECK-GI-NEXT:    neg v3.2d, v3.2d
; CHECK-GI-NEXT:    ushl v5.2d, v0.2d, v5.2d
; CHECK-GI-NEXT:    ushl v0.2d, v0.2d, v2.2d
; CHECK-GI-NEXT:    ushl v3.2d, v1.2d, v3.2d
; CHECK-GI-NEXT:    ushl v1.2d, v1.2d, v4.2d
; CHECK-GI-NEXT:    orr v0.16b, v5.16b, v0.16b
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i64> @llvm.fshr(<4 x i64> %a, <4 x i64> %a, <4 x i64> %c)
  ret <4 x i64> %d
}

define <2 x i128> @rotl_v2i128(<2 x i128> %a, <2 x i128> %c) {
; CHECK-SD-LABEL: rotl_v2i128:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    tst x4, #0x40
; CHECK-SD-NEXT:    mvn w9, w4
; CHECK-SD-NEXT:    csel x8, x1, x0, ne
; CHECK-SD-NEXT:    csel x10, x0, x1, ne
; CHECK-SD-NEXT:    tst x6, #0x40
; CHECK-SD-NEXT:    lsl x11, x8, x4
; CHECK-SD-NEXT:    lsr x12, x10, #1
; CHECK-SD-NEXT:    lsr x8, x8, #1
; CHECK-SD-NEXT:    csel x13, x3, x2, ne
; CHECK-SD-NEXT:    csel x14, x2, x3, ne
; CHECK-SD-NEXT:    lsl x10, x10, x4
; CHECK-SD-NEXT:    lsr x15, x14, #1
; CHECK-SD-NEXT:    lsr x16, x13, #1
; CHECK-SD-NEXT:    lsr x12, x12, x9
; CHECK-SD-NEXT:    lsr x8, x8, x9
; CHECK-SD-NEXT:    lsl x9, x13, x6
; CHECK-SD-NEXT:    mvn w13, w6
; CHECK-SD-NEXT:    lsr x15, x15, x13
; CHECK-SD-NEXT:    lsl x14, x14, x6
; CHECK-SD-NEXT:    lsr x13, x16, x13
; CHECK-SD-NEXT:    orr x0, x11, x12
; CHECK-SD-NEXT:    orr x1, x10, x8
; CHECK-SD-NEXT:    orr x2, x9, x15
; CHECK-SD-NEXT:    orr x3, x14, x13
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v2i128:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    and x8, x4, #0x7f
; CHECK-GI-NEXT:    mov w9, #64 // =0x40
; CHECK-GI-NEXT:    neg x13, x4
; CHECK-GI-NEXT:    sub x10, x9, x8
; CHECK-GI-NEXT:    lsl x12, x1, x8
; CHECK-GI-NEXT:    sub x11, x8, #64
; CHECK-GI-NEXT:    lsr x10, x0, x10
; CHECK-GI-NEXT:    lsl x14, x0, x8
; CHECK-GI-NEXT:    lsl x11, x0, x11
; CHECK-GI-NEXT:    cmp x8, #64
; CHECK-GI-NEXT:    neg x15, x6
; CHECK-GI-NEXT:    orr x8, x10, x12
; CHECK-GI-NEXT:    and x10, x6, #0x7f
; CHECK-GI-NEXT:    csel x12, x14, xzr, lo
; CHECK-GI-NEXT:    sub x14, x9, x10
; CHECK-GI-NEXT:    csel x8, x8, x11, lo
; CHECK-GI-NEXT:    tst x4, #0x7f
; CHECK-GI-NEXT:    sub x11, x10, #64
; CHECK-GI-NEXT:    lsl x16, x2, x10
; CHECK-GI-NEXT:    lsr x14, x2, x14
; CHECK-GI-NEXT:    lsl x17, x3, x10
; CHECK-GI-NEXT:    csel x8, x1, x8, eq
; CHECK-GI-NEXT:    lsl x11, x2, x11
; CHECK-GI-NEXT:    cmp x10, #64
; CHECK-GI-NEXT:    and x4, x15, #0x7f
; CHECK-GI-NEXT:    orr x10, x14, x17
; CHECK-GI-NEXT:    csel x14, x16, xzr, lo
; CHECK-GI-NEXT:    and x16, x13, #0x7f
; CHECK-GI-NEXT:    csel x10, x10, x11, lo
; CHECK-GI-NEXT:    sub x11, x9, x16
; CHECK-GI-NEXT:    sub x17, x16, #64
; CHECK-GI-NEXT:    lsr x18, x0, x16
; CHECK-GI-NEXT:    lsl x11, x1, x11
; CHECK-GI-NEXT:    tst x6, #0x7f
; CHECK-GI-NEXT:    lsr x17, x1, x17
; CHECK-GI-NEXT:    csel x10, x3, x10, eq
; CHECK-GI-NEXT:    cmp x16, #64
; CHECK-GI-NEXT:    orr x11, x18, x11
; CHECK-GI-NEXT:    sub x9, x9, x4
; CHECK-GI-NEXT:    lsr x1, x1, x16
; CHECK-GI-NEXT:    csel x11, x11, x17, lo
; CHECK-GI-NEXT:    tst x13, #0x7f
; CHECK-GI-NEXT:    sub x13, x4, #64
; CHECK-GI-NEXT:    lsr x17, x2, x4
; CHECK-GI-NEXT:    lsl x9, x3, x9
; CHECK-GI-NEXT:    csel x11, x0, x11, eq
; CHECK-GI-NEXT:    cmp x16, #64
; CHECK-GI-NEXT:    lsr x13, x3, x13
; CHECK-GI-NEXT:    orr x0, x12, x11
; CHECK-GI-NEXT:    csel x16, x1, xzr, lo
; CHECK-GI-NEXT:    orr x9, x17, x9
; CHECK-GI-NEXT:    cmp x4, #64
; CHECK-GI-NEXT:    lsr x17, x3, x4
; CHECK-GI-NEXT:    csel x9, x9, x13, lo
; CHECK-GI-NEXT:    tst x15, #0x7f
; CHECK-GI-NEXT:    csel x9, x2, x9, eq
; CHECK-GI-NEXT:    cmp x4, #64
; CHECK-GI-NEXT:    orr x1, x8, x16
; CHECK-GI-NEXT:    csel x13, x17, xzr, lo
; CHECK-GI-NEXT:    orr x2, x14, x9
; CHECK-GI-NEXT:    orr x3, x10, x13
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i128> @llvm.fshl(<2 x i128> %a, <2 x i128> %a, <2 x i128> %c)
  ret <2 x i128> %d
}

define <2 x i128> @rotr_v2i128(<2 x i128> %a, <2 x i128> %c) {
; CHECK-SD-LABEL: rotr_v2i128:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    tst x4, #0x40
; CHECK-SD-NEXT:    mvn w9, w4
; CHECK-SD-NEXT:    csel x8, x0, x1, eq
; CHECK-SD-NEXT:    csel x10, x1, x0, eq
; CHECK-SD-NEXT:    tst x6, #0x40
; CHECK-SD-NEXT:    lsr x11, x8, x4
; CHECK-SD-NEXT:    lsl x12, x10, #1
; CHECK-SD-NEXT:    lsl x8, x8, #1
; CHECK-SD-NEXT:    csel x13, x2, x3, eq
; CHECK-SD-NEXT:    csel x14, x3, x2, eq
; CHECK-SD-NEXT:    lsr x10, x10, x4
; CHECK-SD-NEXT:    lsl x15, x14, #1
; CHECK-SD-NEXT:    lsl x16, x13, #1
; CHECK-SD-NEXT:    lsl x12, x12, x9
; CHECK-SD-NEXT:    lsl x8, x8, x9
; CHECK-SD-NEXT:    lsr x9, x13, x6
; CHECK-SD-NEXT:    mvn w13, w6
; CHECK-SD-NEXT:    lsl x15, x15, x13
; CHECK-SD-NEXT:    lsr x14, x14, x6
; CHECK-SD-NEXT:    lsl x13, x16, x13
; CHECK-SD-NEXT:    orr x0, x12, x11
; CHECK-SD-NEXT:    orr x1, x8, x10
; CHECK-SD-NEXT:    orr x2, x15, x9
; CHECK-SD-NEXT:    orr x3, x13, x14
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v2i128:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    and x8, x4, #0x7f
; CHECK-GI-NEXT:    mov w9, #64 // =0x40
; CHECK-GI-NEXT:    and x14, x6, #0x7f
; CHECK-GI-NEXT:    sub x10, x9, x8
; CHECK-GI-NEXT:    sub x11, x8, #64
; CHECK-GI-NEXT:    lsr x12, x0, x8
; CHECK-GI-NEXT:    lsl x10, x1, x10
; CHECK-GI-NEXT:    lsr x11, x1, x11
; CHECK-GI-NEXT:    cmp x8, #64
; CHECK-GI-NEXT:    sub x15, x9, x14
; CHECK-GI-NEXT:    neg x13, x4
; CHECK-GI-NEXT:    lsr x17, x3, x14
; CHECK-GI-NEXT:    orr x10, x12, x10
; CHECK-GI-NEXT:    lsr x12, x1, x8
; CHECK-GI-NEXT:    lsl x15, x3, x15
; CHECK-GI-NEXT:    csel x10, x10, x11, lo
; CHECK-GI-NEXT:    tst x4, #0x7f
; CHECK-GI-NEXT:    sub x11, x14, #64
; CHECK-GI-NEXT:    csel x10, x0, x10, eq
; CHECK-GI-NEXT:    cmp x8, #64
; CHECK-GI-NEXT:    lsr x8, x2, x14
; CHECK-GI-NEXT:    lsr x11, x3, x11
; CHECK-GI-NEXT:    csel x12, x12, xzr, lo
; CHECK-GI-NEXT:    cmp x14, #64
; CHECK-GI-NEXT:    orr x8, x8, x15
; CHECK-GI-NEXT:    neg x16, x6
; CHECK-GI-NEXT:    csel x8, x8, x11, lo
; CHECK-GI-NEXT:    tst x6, #0x7f
; CHECK-GI-NEXT:    and x11, x13, #0x7f
; CHECK-GI-NEXT:    csel x8, x2, x8, eq
; CHECK-GI-NEXT:    cmp x14, #64
; CHECK-GI-NEXT:    sub x14, x9, x11
; CHECK-GI-NEXT:    sub x15, x11, #64
; CHECK-GI-NEXT:    lsr x14, x0, x14
; CHECK-GI-NEXT:    lsl x18, x1, x11
; CHECK-GI-NEXT:    lsl x4, x0, x11
; CHECK-GI-NEXT:    lsl x15, x0, x15
; CHECK-GI-NEXT:    and x0, x16, #0x7f
; CHECK-GI-NEXT:    csel x17, x17, xzr, lo
; CHECK-GI-NEXT:    orr x14, x14, x18
; CHECK-GI-NEXT:    cmp x11, #64
; CHECK-GI-NEXT:    sub x9, x9, x0
; CHECK-GI-NEXT:    csel x14, x14, x15, lo
; CHECK-GI-NEXT:    sub x15, x0, #64
; CHECK-GI-NEXT:    lsr x9, x2, x9
; CHECK-GI-NEXT:    lsl x18, x3, x0
; CHECK-GI-NEXT:    csel x11, x4, xzr, lo
; CHECK-GI-NEXT:    tst x13, #0x7f
; CHECK-GI-NEXT:    lsl x13, x2, x0
; CHECK-GI-NEXT:    lsl x15, x2, x15
; CHECK-GI-NEXT:    csel x14, x1, x14, eq
; CHECK-GI-NEXT:    orr x9, x9, x18
; CHECK-GI-NEXT:    cmp x0, #64
; CHECK-GI-NEXT:    csel x13, x13, xzr, lo
; CHECK-GI-NEXT:    csel x9, x9, x15, lo
; CHECK-GI-NEXT:    tst x16, #0x7f
; CHECK-GI-NEXT:    csel x9, x3, x9, eq
; CHECK-GI-NEXT:    orr x0, x10, x11
; CHECK-GI-NEXT:    orr x1, x12, x14
; CHECK-GI-NEXT:    orr x2, x8, x13
; CHECK-GI-NEXT:    orr x3, x17, x9
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i128> @llvm.fshr(<2 x i128> %a, <2 x i128> %a, <2 x i128> %c)
  ret <2 x i128> %d
}

define <8 x i8> @fshl_v8i8(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c) {
; CHECK-LABEL: fshl_v8i8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v3.8b, #7
; CHECK-NEXT:    ushr v1.8b, v1.8b, #1
; CHECK-NEXT:    bic v4.8b, v3.8b, v2.8b
; CHECK-NEXT:    and v2.8b, v2.8b, v3.8b
; CHECK-NEXT:    neg v3.8b, v4.8b
; CHECK-NEXT:    ushl v0.8b, v0.8b, v2.8b
; CHECK-NEXT:    ushl v1.8b, v1.8b, v3.8b
; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-NEXT:    ret
entry:
  %d = call <8 x i8> @llvm.fshl(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c)
  ret <8 x i8> %d
}

define <8 x i8> @fshr_v8i8(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c) {
; CHECK-SD-LABEL: fshr_v8i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v3.8b, #7
; CHECK-SD-NEXT:    add v0.8b, v0.8b, v0.8b
; CHECK-SD-NEXT:    and v4.8b, v2.8b, v3.8b
; CHECK-SD-NEXT:    bic v2.8b, v3.8b, v2.8b
; CHECK-SD-NEXT:    neg v3.8b, v4.8b
; CHECK-SD-NEXT:    ushl v0.8b, v0.8b, v2.8b
; CHECK-SD-NEXT:    ushl v1.8b, v1.8b, v3.8b
; CHECK-SD-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v8i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v3.8b, #7
; CHECK-GI-NEXT:    shl v0.8b, v0.8b, #1
; CHECK-GI-NEXT:    and v4.8b, v2.8b, v3.8b
; CHECK-GI-NEXT:    bic v2.8b, v3.8b, v2.8b
; CHECK-GI-NEXT:    neg v3.8b, v4.8b
; CHECK-GI-NEXT:    ushl v0.8b, v0.8b, v2.8b
; CHECK-GI-NEXT:    ushl v1.8b, v1.8b, v3.8b
; CHECK-GI-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i8> @llvm.fshr(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c)
  ret <8 x i8> %d
}

define <16 x i8> @fshl_v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
; CHECK-LABEL: fshl_v16i8:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v3.16b, #7
; CHECK-NEXT:    ushr v1.16b, v1.16b, #1
; CHECK-NEXT:    bic v4.16b, v3.16b, v2.16b
; CHECK-NEXT:    and v2.16b, v2.16b, v3.16b
; CHECK-NEXT:    neg v3.16b, v4.16b
; CHECK-NEXT:    ushl v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    ushl v1.16b, v1.16b, v3.16b
; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    ret
entry:
  %d = call <16 x i8> @llvm.fshl(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c)
  ret <16 x i8> %d
}

define <16 x i8> @fshr_v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
; CHECK-SD-LABEL: fshr_v16i8:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v3.16b, #7
; CHECK-SD-NEXT:    add v0.16b, v0.16b, v0.16b
; CHECK-SD-NEXT:    and v4.16b, v2.16b, v3.16b
; CHECK-SD-NEXT:    bic v2.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    neg v3.16b, v4.16b
; CHECK-SD-NEXT:    ushl v0.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    ushl v1.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v16i8:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v3.16b, #7
; CHECK-GI-NEXT:    shl v0.16b, v0.16b, #1
; CHECK-GI-NEXT:    and v4.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    bic v2.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    neg v3.16b, v4.16b
; CHECK-GI-NEXT:    ushl v0.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    ushl v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <16 x i8> @llvm.fshr(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c)
  ret <16 x i8> %d
}

define <4 x i16> @fshl_v4i16(<4 x i16> %a, <4 x i16> %b, <4 x i16> %c) {
; CHECK-LABEL: fshl_v4i16:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v3.4h, #15
; CHECK-NEXT:    ushr v1.4h, v1.4h, #1
; CHECK-NEXT:    bic v4.8b, v3.8b, v2.8b
; CHECK-NEXT:    and v2.8b, v2.8b, v3.8b
; CHECK-NEXT:    neg v3.4h, v4.4h
; CHECK-NEXT:    ushl v0.4h, v0.4h, v2.4h
; CHECK-NEXT:    ushl v1.4h, v1.4h, v3.4h
; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-NEXT:    ret
entry:
  %d = call <4 x i16> @llvm.fshl(<4 x i16> %a, <4 x i16> %b, <4 x i16> %c)
  ret <4 x i16> %d
}

define <4 x i16> @fshr_v4i16(<4 x i16> %a, <4 x i16> %b, <4 x i16> %c) {
; CHECK-SD-LABEL: fshr_v4i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v3.4h, #15
; CHECK-SD-NEXT:    add v0.4h, v0.4h, v0.4h
; CHECK-SD-NEXT:    and v4.8b, v2.8b, v3.8b
; CHECK-SD-NEXT:    bic v2.8b, v3.8b, v2.8b
; CHECK-SD-NEXT:    neg v3.4h, v4.4h
; CHECK-SD-NEXT:    ushl v0.4h, v0.4h, v2.4h
; CHECK-SD-NEXT:    ushl v1.4h, v1.4h, v3.4h
; CHECK-SD-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v4i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v3.4h, #15
; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #1
; CHECK-GI-NEXT:    and v4.8b, v2.8b, v3.8b
; CHECK-GI-NEXT:    bic v2.8b, v3.8b, v2.8b
; CHECK-GI-NEXT:    neg v3.4h, v4.4h
; CHECK-GI-NEXT:    ushl v0.4h, v0.4h, v2.4h
; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v3.4h
; CHECK-GI-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i16> @llvm.fshr(<4 x i16> %a, <4 x i16> %b, <4 x i16> %c)
  ret <4 x i16> %d
}

define <7 x i16> @fshl_v7i16(<7 x i16> %a, <7 x i16> %b, <7 x i16> %c) {
; CHECK-SD-LABEL: fshl_v7i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v3.8h, #15
; CHECK-SD-NEXT:    ushr v1.8h, v1.8h, #1
; CHECK-SD-NEXT:    bic v4.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    and v2.16b, v2.16b, v3.16b
; CHECK-SD-NEXT:    neg v3.8h, v4.8h
; CHECK-SD-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    ushl v1.8h, v1.8h, v3.8h
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v7i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #65535 // =0xffff
; CHECK-GI-NEXT:    mov w9, #15 // =0xf
; CHECK-GI-NEXT:    mov w10, #1 // =0x1
; CHECK-GI-NEXT:    fmov s3, w8
; CHECK-GI-NEXT:    fmov s4, w9
; CHECK-GI-NEXT:    fmov s5, w10
; CHECK-GI-NEXT:    mov v3.h[1], w8
; CHECK-GI-NEXT:    mov v4.h[1], w9
; CHECK-GI-NEXT:    mov v5.h[1], w10
; CHECK-GI-NEXT:    mov v3.h[2], w8
; CHECK-GI-NEXT:    mov v4.h[2], w9
; CHECK-GI-NEXT:    mov v5.h[2], w10
; CHECK-GI-NEXT:    mov v3.h[3], w8
; CHECK-GI-NEXT:    mov v4.h[3], w9
; CHECK-GI-NEXT:    mov v5.h[3], w10
; CHECK-GI-NEXT:    mov v3.h[4], w8
; CHECK-GI-NEXT:    mov v4.h[4], w9
; CHECK-GI-NEXT:    mov v5.h[4], w10
; CHECK-GI-NEXT:    mov v3.h[5], w8
; CHECK-GI-NEXT:    mov v4.h[5], w9
; CHECK-GI-NEXT:    mov v5.h[5], w10
; CHECK-GI-NEXT:    mov v3.h[6], w8
; CHECK-GI-NEXT:    mov v4.h[6], w9
; CHECK-GI-NEXT:    mov v5.h[6], w10
; CHECK-GI-NEXT:    eor v3.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    neg v5.8h, v5.8h
; CHECK-GI-NEXT:    and v2.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    and v3.16b, v3.16b, v4.16b
; CHECK-GI-NEXT:    ushl v1.8h, v1.8h, v5.8h
; CHECK-GI-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    neg v3.8h, v3.8h
; CHECK-GI-NEXT:    ushl v1.8h, v1.8h, v3.8h
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i16> @llvm.fshl(<7 x i16> %a, <7 x i16> %b, <7 x i16> %c)
  ret <7 x i16> %d
}

define <7 x i16> @fshr_v7i16(<7 x i16> %a, <7 x i16> %b, <7 x i16> %c) {
; CHECK-SD-LABEL: fshr_v7i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v3.8h, #15
; CHECK-SD-NEXT:    add v0.8h, v0.8h, v0.8h
; CHECK-SD-NEXT:    and v4.16b, v2.16b, v3.16b
; CHECK-SD-NEXT:    bic v2.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    neg v3.8h, v4.8h
; CHECK-SD-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    ushl v1.8h, v1.8h, v3.8h
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v7i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #15 // =0xf
; CHECK-GI-NEXT:    mov w9, #65535 // =0xffff
; CHECK-GI-NEXT:    mov w10, #1 // =0x1
; CHECK-GI-NEXT:    fmov s3, w8
; CHECK-GI-NEXT:    fmov s4, w9
; CHECK-GI-NEXT:    fmov s5, w10
; CHECK-GI-NEXT:    mov v3.h[1], w8
; CHECK-GI-NEXT:    mov v4.h[1], w9
; CHECK-GI-NEXT:    mov v5.h[1], w10
; CHECK-GI-NEXT:    mov v3.h[2], w8
; CHECK-GI-NEXT:    mov v4.h[2], w9
; CHECK-GI-NEXT:    mov v5.h[2], w10
; CHECK-GI-NEXT:    mov v3.h[3], w8
; CHECK-GI-NEXT:    mov v4.h[3], w9
; CHECK-GI-NEXT:    mov v5.h[3], w10
; CHECK-GI-NEXT:    mov v3.h[4], w8
; CHECK-GI-NEXT:    mov v4.h[4], w9
; CHECK-GI-NEXT:    mov v5.h[4], w10
; CHECK-GI-NEXT:    mov v3.h[5], w8
; CHECK-GI-NEXT:    mov v4.h[5], w9
; CHECK-GI-NEXT:    mov v5.h[5], w10
; CHECK-GI-NEXT:    mov v3.h[6], w8
; CHECK-GI-NEXT:    mov v4.h[6], w9
; CHECK-GI-NEXT:    mov v5.h[6], w10
; CHECK-GI-NEXT:    eor v4.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    and v2.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    ushl v0.8h, v0.8h, v5.8h
; CHECK-GI-NEXT:    and v3.16b, v4.16b, v3.16b
; CHECK-GI-NEXT:    neg v2.8h, v2.8h
; CHECK-GI-NEXT:    ushl v0.8h, v0.8h, v3.8h
; CHECK-GI-NEXT:    ushl v1.8h, v1.8h, v2.8h
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i16> @llvm.fshr(<7 x i16> %a, <7 x i16> %b, <7 x i16> %c)
  ret <7 x i16> %d
}

define <8 x i16> @fshl_v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
; CHECK-LABEL: fshl_v8i16:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v3.8h, #15
; CHECK-NEXT:    ushr v1.8h, v1.8h, #1
; CHECK-NEXT:    bic v4.16b, v3.16b, v2.16b
; CHECK-NEXT:    and v2.16b, v2.16b, v3.16b
; CHECK-NEXT:    neg v3.8h, v4.8h
; CHECK-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-NEXT:    ushl v1.8h, v1.8h, v3.8h
; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    ret
entry:
  %d = call <8 x i16> @llvm.fshl(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c)
  ret <8 x i16> %d
}

define <8 x i16> @fshr_v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
; CHECK-SD-LABEL: fshr_v8i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v3.8h, #15
; CHECK-SD-NEXT:    add v0.8h, v0.8h, v0.8h
; CHECK-SD-NEXT:    and v4.16b, v2.16b, v3.16b
; CHECK-SD-NEXT:    bic v2.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    neg v3.8h, v4.8h
; CHECK-SD-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    ushl v1.8h, v1.8h, v3.8h
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v8i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v3.8h, #15
; CHECK-GI-NEXT:    shl v0.8h, v0.8h, #1
; CHECK-GI-NEXT:    and v4.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    bic v2.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    neg v3.8h, v4.8h
; CHECK-GI-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    ushl v1.8h, v1.8h, v3.8h
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i16> @llvm.fshr(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c)
  ret <8 x i16> %d
}

define <16 x i16> @fshl_v16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16> %c) {
; CHECK-LABEL: fshl_v16i16:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v6.8h, #15
; CHECK-NEXT:    ushr v2.8h, v2.8h, #1
; CHECK-NEXT:    ushr v3.8h, v3.8h, #1
; CHECK-NEXT:    bic v7.16b, v6.16b, v4.16b
; CHECK-NEXT:    bic v16.16b, v6.16b, v5.16b
; CHECK-NEXT:    and v4.16b, v4.16b, v6.16b
; CHECK-NEXT:    and v5.16b, v5.16b, v6.16b
; CHECK-NEXT:    neg v6.8h, v7.8h
; CHECK-NEXT:    neg v7.8h, v16.8h
; CHECK-NEXT:    ushl v0.8h, v0.8h, v4.8h
; CHECK-NEXT:    ushl v1.8h, v1.8h, v5.8h
; CHECK-NEXT:    ushl v2.8h, v2.8h, v6.8h
; CHECK-NEXT:    ushl v3.8h, v3.8h, v7.8h
; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    orr v1.16b, v1.16b, v3.16b
; CHECK-NEXT:    ret
entry:
  %d = call <16 x i16> @llvm.fshl(<16 x i16> %a, <16 x i16> %b, <16 x i16> %c)
  ret <16 x i16> %d
}

define <16 x i16> @fshr_v16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16> %c) {
; CHECK-SD-LABEL: fshr_v16i16:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v6.8h, #15
; CHECK-SD-NEXT:    add v0.8h, v0.8h, v0.8h
; CHECK-SD-NEXT:    add v1.8h, v1.8h, v1.8h
; CHECK-SD-NEXT:    and v7.16b, v4.16b, v6.16b
; CHECK-SD-NEXT:    and v16.16b, v5.16b, v6.16b
; CHECK-SD-NEXT:    bic v4.16b, v6.16b, v4.16b
; CHECK-SD-NEXT:    bic v5.16b, v6.16b, v5.16b
; CHECK-SD-NEXT:    neg v6.8h, v7.8h
; CHECK-SD-NEXT:    neg v7.8h, v16.8h
; CHECK-SD-NEXT:    ushl v0.8h, v0.8h, v4.8h
; CHECK-SD-NEXT:    ushl v1.8h, v1.8h, v5.8h
; CHECK-SD-NEXT:    ushl v2.8h, v2.8h, v6.8h
; CHECK-SD-NEXT:    ushl v3.8h, v3.8h, v7.8h
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v16i16:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v6.8h, #15
; CHECK-GI-NEXT:    shl v0.8h, v0.8h, #1
; CHECK-GI-NEXT:    shl v1.8h, v1.8h, #1
; CHECK-GI-NEXT:    and v7.16b, v4.16b, v6.16b
; CHECK-GI-NEXT:    and v16.16b, v5.16b, v6.16b
; CHECK-GI-NEXT:    bic v4.16b, v6.16b, v4.16b
; CHECK-GI-NEXT:    bic v5.16b, v6.16b, v5.16b
; CHECK-GI-NEXT:    neg v6.8h, v7.8h
; CHECK-GI-NEXT:    neg v7.8h, v16.8h
; CHECK-GI-NEXT:    ushl v0.8h, v0.8h, v4.8h
; CHECK-GI-NEXT:    ushl v1.8h, v1.8h, v5.8h
; CHECK-GI-NEXT:    ushl v2.8h, v2.8h, v6.8h
; CHECK-GI-NEXT:    ushl v3.8h, v3.8h, v7.8h
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <16 x i16> @llvm.fshr(<16 x i16> %a, <16 x i16> %b, <16 x i16> %c)
  ret <16 x i16> %d
}

define <2 x i32> @fshl_v2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) {
; CHECK-LABEL: fshl_v2i32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v3.2s, #31
; CHECK-NEXT:    ushr v1.2s, v1.2s, #1
; CHECK-NEXT:    bic v4.8b, v3.8b, v2.8b
; CHECK-NEXT:    and v2.8b, v2.8b, v3.8b
; CHECK-NEXT:    neg v3.2s, v4.2s
; CHECK-NEXT:    ushl v0.2s, v0.2s, v2.2s
; CHECK-NEXT:    ushl v1.2s, v1.2s, v3.2s
; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-NEXT:    ret
entry:
  %d = call <2 x i32> @llvm.fshl(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c)
  ret <2 x i32> %d
}

define <2 x i32> @fshr_v2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) {
; CHECK-SD-LABEL: fshr_v2i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v3.2s, #31
; CHECK-SD-NEXT:    add v0.2s, v0.2s, v0.2s
; CHECK-SD-NEXT:    and v4.8b, v2.8b, v3.8b
; CHECK-SD-NEXT:    bic v2.8b, v3.8b, v2.8b
; CHECK-SD-NEXT:    neg v3.2s, v4.2s
; CHECK-SD-NEXT:    ushl v0.2s, v0.2s, v2.2s
; CHECK-SD-NEXT:    ushl v1.2s, v1.2s, v3.2s
; CHECK-SD-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v2i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v3.2s, #31
; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #1
; CHECK-GI-NEXT:    and v4.8b, v2.8b, v3.8b
; CHECK-GI-NEXT:    bic v2.8b, v3.8b, v2.8b
; CHECK-GI-NEXT:    neg v3.2s, v4.2s
; CHECK-GI-NEXT:    ushl v0.2s, v0.2s, v2.2s
; CHECK-GI-NEXT:    ushl v1.2s, v1.2s, v3.2s
; CHECK-GI-NEXT:    orr v0.8b, v0.8b, v1.8b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i32> @llvm.fshr(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c)
  ret <2 x i32> %d
}

define <4 x i32> @fshl_v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
; CHECK-LABEL: fshl_v4i32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v3.4s, #31
; CHECK-NEXT:    ushr v1.4s, v1.4s, #1
; CHECK-NEXT:    bic v4.16b, v3.16b, v2.16b
; CHECK-NEXT:    and v2.16b, v2.16b, v3.16b
; CHECK-NEXT:    neg v3.4s, v4.4s
; CHECK-NEXT:    ushl v0.4s, v0.4s, v2.4s
; CHECK-NEXT:    ushl v1.4s, v1.4s, v3.4s
; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-NEXT:    ret
entry:
  %d = call <4 x i32> @llvm.fshl(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
  ret <4 x i32> %d
}

define <4 x i32> @fshr_v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
; CHECK-SD-LABEL: fshr_v4i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v3.4s, #31
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v0.4s
; CHECK-SD-NEXT:    and v4.16b, v2.16b, v3.16b
; CHECK-SD-NEXT:    bic v2.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    neg v3.4s, v4.4s
; CHECK-SD-NEXT:    ushl v0.4s, v0.4s, v2.4s
; CHECK-SD-NEXT:    ushl v1.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v4i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v3.4s, #31
; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #1
; CHECK-GI-NEXT:    and v4.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    bic v2.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    neg v3.4s, v4.4s
; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v2.4s
; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v3.4s
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i32> @llvm.fshr(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
  ret <4 x i32> %d
}

define <7 x i32> @fshl_v7i32(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c) {
; CHECK-SD-LABEL: fshl_v7i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldr s0, [sp, #48]
; CHECK-SD-NEXT:    add x8, sp, #56
; CHECK-SD-NEXT:    fmov s3, w7
; CHECK-SD-NEXT:    fmov s4, w0
; CHECK-SD-NEXT:    ldr s2, [sp, #80]
; CHECK-SD-NEXT:    mov x9, sp
; CHECK-SD-NEXT:    ld1 { v0.s }[1], [x8]
; CHECK-SD-NEXT:    add x10, sp, #88
; CHECK-SD-NEXT:    ldr s1, [sp, #24]
; CHECK-SD-NEXT:    ld1 { v3.s }[1], [x9]
; CHECK-SD-NEXT:    add x9, sp, #64
; CHECK-SD-NEXT:    ld1 { v2.s }[1], [x10]
; CHECK-SD-NEXT:    mov v4.s[1], w1
; CHECK-SD-NEXT:    add x8, sp, #32
; CHECK-SD-NEXT:    fmov s5, w4
; CHECK-SD-NEXT:    ld1 { v0.s }[2], [x9]
; CHECK-SD-NEXT:    ld1 { v1.s }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #8
; CHECK-SD-NEXT:    add x9, sp, #72
; CHECK-SD-NEXT:    movi v6.4s, #31
; CHECK-SD-NEXT:    add x10, sp, #96
; CHECK-SD-NEXT:    ld1 { v3.s }[2], [x8]
; CHECK-SD-NEXT:    ld1 { v2.s }[2], [x10]
; CHECK-SD-NEXT:    mov v5.s[1], w5
; CHECK-SD-NEXT:    ld1 { v0.s }[3], [x9]
; CHECK-SD-NEXT:    mov v4.s[2], w2
; CHECK-SD-NEXT:    add x8, sp, #16
; CHECK-SD-NEXT:    add x9, sp, #40
; CHECK-SD-NEXT:    ld1 { v3.s }[3], [x8]
; CHECK-SD-NEXT:    ld1 { v1.s }[2], [x9]
; CHECK-SD-NEXT:    bic v16.16b, v6.16b, v2.16b
; CHECK-SD-NEXT:    bic v7.16b, v6.16b, v0.16b
; CHECK-SD-NEXT:    mov v5.s[2], w6
; CHECK-SD-NEXT:    and v0.16b, v0.16b, v6.16b
; CHECK-SD-NEXT:    mov v4.s[3], w3
; CHECK-SD-NEXT:    and v2.16b, v2.16b, v6.16b
; CHECK-SD-NEXT:    ushr v3.4s, v3.4s, #1
; CHECK-SD-NEXT:    ushr v1.4s, v1.4s, #1
; CHECK-SD-NEXT:    neg v6.4s, v16.4s
; CHECK-SD-NEXT:    neg v7.4s, v7.4s
; CHECK-SD-NEXT:    ushl v2.4s, v5.4s, v2.4s
; CHECK-SD-NEXT:    ushl v0.4s, v4.4s, v0.4s
; CHECK-SD-NEXT:    ushl v1.4s, v1.4s, v6.4s
; CHECK-SD-NEXT:    ushl v3.4s, v3.4s, v7.4s
; CHECK-SD-NEXT:    orr v1.16b, v2.16b, v1.16b
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v3.16b
; CHECK-SD-NEXT:    mov w5, v1.s[1]
; CHECK-SD-NEXT:    mov w6, v1.s[2]
; CHECK-SD-NEXT:    fmov w4, s1
; CHECK-SD-NEXT:    mov w1, v0.s[1]
; CHECK-SD-NEXT:    mov w2, v0.s[2]
; CHECK-SD-NEXT:    mov w3, v0.s[3]
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v7i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w11, #-1 // =0xffffffff
; CHECK-GI-NEXT:    ldr s3, [sp, #48]
; CHECK-GI-NEXT:    ldr s19, [sp, #56]
; CHECK-GI-NEXT:    mov v20.s[0], w11
; CHECK-GI-NEXT:    mov v21.s[0], w7
; CHECK-GI-NEXT:    ldr s16, [sp, #80]
; CHECK-GI-NEXT:    ldr s22, [sp, #88]
; CHECK-GI-NEXT:    mov w12, #31 // =0x1f
; CHECK-GI-NEXT:    mov w13, #1 // =0x1
; CHECK-GI-NEXT:    ldr s6, [sp]
; CHECK-GI-NEXT:    mov v3.s[1], v19.s[0]
; CHECK-GI-NEXT:    mov v19.s[0], w12
; CHECK-GI-NEXT:    mov v23.s[0], w13
; CHECK-GI-NEXT:    mov v16.s[1], v22.s[0]
; CHECK-GI-NEXT:    ldr s7, [sp, #64]
; CHECK-GI-NEXT:    mov v20.s[1], w11
; CHECK-GI-NEXT:    mov v22.s[0], w0
; CHECK-GI-NEXT:    mov v21.s[1], v6.s[0]
; CHECK-GI-NEXT:    ldr s24, [sp, #96]
; CHECK-GI-NEXT:    mov v6.s[0], w12
; CHECK-GI-NEXT:    ldr s5, [sp, #8]
; CHECK-GI-NEXT:    ldr s18, [sp, #48]
; CHECK-GI-NEXT:    mov v3.s[2], v7.s[0]
; CHECK-GI-NEXT:    mov v19.s[1], w12
; CHECK-GI-NEXT:    mov v23.s[1], w13
; CHECK-GI-NEXT:    ldr s0, [sp, #24]
; CHECK-GI-NEXT:    ldr s4, [sp, #32]
; CHECK-GI-NEXT:    add x10, sp, #56
; CHECK-GI-NEXT:    mov v16.s[2], v24.s[0]
; CHECK-GI-NEXT:    mov v20.s[2], w11
; CHECK-GI-NEXT:    ldr s17, [sp, #72]
; CHECK-GI-NEXT:    ld1 { v18.s }[1], [x10]
; CHECK-GI-NEXT:    mov v22.s[1], w1
; CHECK-GI-NEXT:    mov v21.s[2], v5.s[0]
; CHECK-GI-NEXT:    mov v5.s[0], w4
; CHECK-GI-NEXT:    ldr s24, [sp, #80]
; CHECK-GI-NEXT:    mov v6.s[1], w12
; CHECK-GI-NEXT:    mov v0.s[1], v4.s[0]
; CHECK-GI-NEXT:    add x9, sp, #64
; CHECK-GI-NEXT:    add x10, sp, #88
; CHECK-GI-NEXT:    movi v7.4s, #31
; CHECK-GI-NEXT:    mov v3.s[3], v17.s[0]
; CHECK-GI-NEXT:    mov v19.s[2], w12
; CHECK-GI-NEXT:    mov v23.s[2], w13
; CHECK-GI-NEXT:    ldr s2, [sp, #16]
; CHECK-GI-NEXT:    ldr s1, [sp, #40]
; CHECK-GI-NEXT:    ld1 { v18.s }[2], [x9]
; CHECK-GI-NEXT:    ld1 { v24.s }[1], [x10]
; CHECK-GI-NEXT:    eor v4.16b, v16.16b, v20.16b
; CHECK-GI-NEXT:    mov v22.s[2], w2
; CHECK-GI-NEXT:    mov v5.s[1], w5
; CHECK-GI-NEXT:    add x8, sp, #72
; CHECK-GI-NEXT:    add x9, sp, #96
; CHECK-GI-NEXT:    mov v21.s[3], v2.s[0]
; CHECK-GI-NEXT:    mov v6.s[2], w12
; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
; CHECK-GI-NEXT:    ld1 { v18.s }[3], [x8]
; CHECK-GI-NEXT:    bic v2.16b, v7.16b, v3.16b
; CHECK-GI-NEXT:    ld1 { v24.s }[2], [x9]
; CHECK-GI-NEXT:    and v1.16b, v4.16b, v19.16b
; CHECK-GI-NEXT:    neg v3.4s, v23.4s
; CHECK-GI-NEXT:    mov v22.s[3], w3
; CHECK-GI-NEXT:    mov v5.s[2], w6
; CHECK-GI-NEXT:    and v4.16b, v18.16b, v7.16b
; CHECK-GI-NEXT:    ushr v7.4s, v21.4s, #1
; CHECK-GI-NEXT:    neg v2.4s, v2.4s
; CHECK-GI-NEXT:    and v6.16b, v24.16b, v6.16b
; CHECK-GI-NEXT:    neg v1.4s, v1.4s
; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v3.4s
; CHECK-GI-NEXT:    ushl v3.4s, v22.4s, v4.4s
; CHECK-GI-NEXT:    ushl v2.4s, v7.4s, v2.4s
; CHECK-GI-NEXT:    ushl v4.4s, v5.4s, v6.4s
; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v1.4s
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    orr v0.16b, v4.16b, v0.16b
; CHECK-GI-NEXT:    mov s2, v1.s[1]
; CHECK-GI-NEXT:    mov s3, v1.s[2]
; CHECK-GI-NEXT:    mov s4, v1.s[3]
; CHECK-GI-NEXT:    mov s5, v0.s[1]
; CHECK-GI-NEXT:    mov s6, v0.s[2]
; CHECK-GI-NEXT:    fmov w0, s1
; CHECK-GI-NEXT:    fmov w4, s0
; CHECK-GI-NEXT:    fmov w1, s2
; CHECK-GI-NEXT:    fmov w2, s3
; CHECK-GI-NEXT:    fmov w3, s4
; CHECK-GI-NEXT:    fmov w5, s5
; CHECK-GI-NEXT:    fmov w6, s6
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i32> @llvm.fshl(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c)
  ret <7 x i32> %d
}

define <7 x i32> @fshr_v7i32(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c) {
; CHECK-SD-LABEL: fshr_v7i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    fmov s1, w0
; CHECK-SD-NEXT:    ldr s0, [sp, #48]
; CHECK-SD-NEXT:    add x8, sp, #56
; CHECK-SD-NEXT:    ldr s2, [sp, #80]
; CHECK-SD-NEXT:    fmov s3, w4
; CHECK-SD-NEXT:    add x9, sp, #64
; CHECK-SD-NEXT:    ld1 { v0.s }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #88
; CHECK-SD-NEXT:    fmov s4, w7
; CHECK-SD-NEXT:    mov v1.s[1], w1
; CHECK-SD-NEXT:    ld1 { v2.s }[1], [x8]
; CHECK-SD-NEXT:    mov x10, sp
; CHECK-SD-NEXT:    mov v3.s[1], w5
; CHECK-SD-NEXT:    add x8, sp, #72
; CHECK-SD-NEXT:    movi v5.4s, #31
; CHECK-SD-NEXT:    ld1 { v0.s }[2], [x9]
; CHECK-SD-NEXT:    add x9, sp, #96
; CHECK-SD-NEXT:    ld1 { v4.s }[1], [x10]
; CHECK-SD-NEXT:    ld1 { v2.s }[2], [x9]
; CHECK-SD-NEXT:    ldr s6, [sp, #24]
; CHECK-SD-NEXT:    add x9, sp, #8
; CHECK-SD-NEXT:    mov v1.s[2], w2
; CHECK-SD-NEXT:    ld1 { v0.s }[3], [x8]
; CHECK-SD-NEXT:    add x8, sp, #32
; CHECK-SD-NEXT:    mov v3.s[2], w6
; CHECK-SD-NEXT:    ld1 { v4.s }[2], [x9]
; CHECK-SD-NEXT:    ld1 { v6.s }[1], [x8]
; CHECK-SD-NEXT:    bic v16.16b, v5.16b, v2.16b
; CHECK-SD-NEXT:    and v2.16b, v2.16b, v5.16b
; CHECK-SD-NEXT:    add x8, sp, #40
; CHECK-SD-NEXT:    add x9, sp, #16
; CHECK-SD-NEXT:    mov v1.s[3], w3
; CHECK-SD-NEXT:    and v7.16b, v0.16b, v5.16b
; CHECK-SD-NEXT:    bic v0.16b, v5.16b, v0.16b
; CHECK-SD-NEXT:    ld1 { v4.s }[3], [x9]
; CHECK-SD-NEXT:    ld1 { v6.s }[2], [x8]
; CHECK-SD-NEXT:    add v3.4s, v3.4s, v3.4s
; CHECK-SD-NEXT:    neg v2.4s, v2.4s
; CHECK-SD-NEXT:    neg v5.4s, v7.4s
; CHECK-SD-NEXT:    add v1.4s, v1.4s, v1.4s
; CHECK-SD-NEXT:    ushl v3.4s, v3.4s, v16.4s
; CHECK-SD-NEXT:    ushl v2.4s, v6.4s, v2.4s
; CHECK-SD-NEXT:    ushl v0.4s, v1.4s, v0.4s
; CHECK-SD-NEXT:    ushl v1.4s, v4.4s, v5.4s
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    orr v1.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    mov w1, v0.s[1]
; CHECK-SD-NEXT:    mov w2, v0.s[2]
; CHECK-SD-NEXT:    mov w3, v0.s[3]
; CHECK-SD-NEXT:    mov w5, v1.s[1]
; CHECK-SD-NEXT:    mov w6, v1.s[2]
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    fmov w4, s1
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v7i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldr s2, [sp, #48]
; CHECK-GI-NEXT:    ldr s16, [sp, #56]
; CHECK-GI-NEXT:    add x10, sp, #56
; CHECK-GI-NEXT:    ldr s4, [sp, #80]
; CHECK-GI-NEXT:    ldr s17, [sp, #88]
; CHECK-GI-NEXT:    mov w11, #-1 // =0xffffffff
; CHECK-GI-NEXT:    ldr s7, [sp, #48]
; CHECK-GI-NEXT:    mov v2.s[1], v16.s[0]
; CHECK-GI-NEXT:    mov v16.s[0], w0
; CHECK-GI-NEXT:    mov v4.s[1], v17.s[0]
; CHECK-GI-NEXT:    ldr s19, [sp, #64]
; CHECK-GI-NEXT:    mov v18.s[0], w11
; CHECK-GI-NEXT:    ld1 { v7.s }[1], [x10]
; CHECK-GI-NEXT:    mov w10, #31 // =0x1f
; CHECK-GI-NEXT:    add x9, sp, #64
; CHECK-GI-NEXT:    mov v17.s[0], w10
; CHECK-GI-NEXT:    ldr s21, [sp, #96]
; CHECK-GI-NEXT:    mov v22.s[0], w4
; CHECK-GI-NEXT:    mov v2.s[2], v19.s[0]
; CHECK-GI-NEXT:    mov v19.s[0], w7
; CHECK-GI-NEXT:    mov v16.s[1], w1
; CHECK-GI-NEXT:    ld1 { v7.s }[2], [x9]
; CHECK-GI-NEXT:    mov w9, #1 // =0x1
; CHECK-GI-NEXT:    mov v4.s[2], v21.s[0]
; CHECK-GI-NEXT:    mov v21.s[0], w10
; CHECK-GI-NEXT:    mov v23.s[0], w9
; CHECK-GI-NEXT:    ldr s6, [sp]
; CHECK-GI-NEXT:    ldr s24, [sp, #80]
; CHECK-GI-NEXT:    mov v17.s[1], w10
; CHECK-GI-NEXT:    mov v18.s[1], w11
; CHECK-GI-NEXT:    add x12, sp, #88
; CHECK-GI-NEXT:    mov v19.s[1], v6.s[0]
; CHECK-GI-NEXT:    add x8, sp, #72
; CHECK-GI-NEXT:    ld1 { v24.s }[1], [x12]
; CHECK-GI-NEXT:    mov v16.s[2], w2
; CHECK-GI-NEXT:    mov v22.s[1], w5
; CHECK-GI-NEXT:    mov v21.s[1], w10
; CHECK-GI-NEXT:    mov v23.s[1], w9
; CHECK-GI-NEXT:    ldr s5, [sp, #8]
; CHECK-GI-NEXT:    ldr s0, [sp, #24]
; CHECK-GI-NEXT:    ldr s3, [sp, #32]
; CHECK-GI-NEXT:    ld1 { v7.s }[3], [x8]
; CHECK-GI-NEXT:    movi v6.4s, #31
; CHECK-GI-NEXT:    add x8, sp, #96
; CHECK-GI-NEXT:    mov v17.s[2], w10
; CHECK-GI-NEXT:    mov v18.s[2], w11
; CHECK-GI-NEXT:    ldr s20, [sp, #72]
; CHECK-GI-NEXT:    ld1 { v24.s }[2], [x8]
; CHECK-GI-NEXT:    mov v19.s[2], v5.s[0]
; CHECK-GI-NEXT:    mov v0.s[1], v3.s[0]
; CHECK-GI-NEXT:    mov v16.s[3], w3
; CHECK-GI-NEXT:    mov v2.s[3], v20.s[0]
; CHECK-GI-NEXT:    mov v21.s[2], w10
; CHECK-GI-NEXT:    mov v22.s[2], w6
; CHECK-GI-NEXT:    mov v23.s[2], w9
; CHECK-GI-NEXT:    ldr s1, [sp, #16]
; CHECK-GI-NEXT:    and v5.16b, v7.16b, v6.16b
; CHECK-GI-NEXT:    ldr s3, [sp, #40]
; CHECK-GI-NEXT:    and v7.16b, v24.16b, v17.16b
; CHECK-GI-NEXT:    eor v4.16b, v4.16b, v18.16b
; CHECK-GI-NEXT:    mov v19.s[3], v1.s[0]
; CHECK-GI-NEXT:    shl v1.4s, v16.4s, #1
; CHECK-GI-NEXT:    mov v0.s[2], v3.s[0]
; CHECK-GI-NEXT:    bic v2.16b, v6.16b, v2.16b
; CHECK-GI-NEXT:    neg v5.4s, v5.4s
; CHECK-GI-NEXT:    and v3.16b, v4.16b, v21.16b
; CHECK-GI-NEXT:    ushl v4.4s, v22.4s, v23.4s
; CHECK-GI-NEXT:    neg v6.4s, v7.4s
; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v2.4s
; CHECK-GI-NEXT:    ushl v2.4s, v19.4s, v5.4s
; CHECK-GI-NEXT:    ushl v3.4s, v4.4s, v3.4s
; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v6.4s
; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v2.16b
; CHECK-GI-NEXT:    orr v0.16b, v3.16b, v0.16b
; CHECK-GI-NEXT:    mov s2, v1.s[1]
; CHECK-GI-NEXT:    mov s3, v1.s[2]
; CHECK-GI-NEXT:    mov s4, v1.s[3]
; CHECK-GI-NEXT:    mov s5, v0.s[1]
; CHECK-GI-NEXT:    mov s6, v0.s[2]
; CHECK-GI-NEXT:    fmov w0, s1
; CHECK-GI-NEXT:    fmov w4, s0
; CHECK-GI-NEXT:    fmov w1, s2
; CHECK-GI-NEXT:    fmov w2, s3
; CHECK-GI-NEXT:    fmov w3, s4
; CHECK-GI-NEXT:    fmov w5, s5
; CHECK-GI-NEXT:    fmov w6, s6
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i32> @llvm.fshr(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c)
  ret <7 x i32> %d
}

define <8 x i32> @fshl_v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c) {
; CHECK-LABEL: fshl_v8i32:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v6.4s, #31
; CHECK-NEXT:    ushr v2.4s, v2.4s, #1
; CHECK-NEXT:    ushr v3.4s, v3.4s, #1
; CHECK-NEXT:    bic v7.16b, v6.16b, v4.16b
; CHECK-NEXT:    bic v16.16b, v6.16b, v5.16b
; CHECK-NEXT:    and v4.16b, v4.16b, v6.16b
; CHECK-NEXT:    and v5.16b, v5.16b, v6.16b
; CHECK-NEXT:    neg v6.4s, v7.4s
; CHECK-NEXT:    neg v7.4s, v16.4s
; CHECK-NEXT:    ushl v0.4s, v0.4s, v4.4s
; CHECK-NEXT:    ushl v1.4s, v1.4s, v5.4s
; CHECK-NEXT:    ushl v2.4s, v2.4s, v6.4s
; CHECK-NEXT:    ushl v3.4s, v3.4s, v7.4s
; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-NEXT:    orr v1.16b, v1.16b, v3.16b
; CHECK-NEXT:    ret
entry:
  %d = call <8 x i32> @llvm.fshl(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c)
  ret <8 x i32> %d
}

define <8 x i32> @fshr_v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c) {
; CHECK-SD-LABEL: fshr_v8i32:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    movi v6.4s, #31
; CHECK-SD-NEXT:    add v0.4s, v0.4s, v0.4s
; CHECK-SD-NEXT:    add v1.4s, v1.4s, v1.4s
; CHECK-SD-NEXT:    and v7.16b, v4.16b, v6.16b
; CHECK-SD-NEXT:    and v16.16b, v5.16b, v6.16b
; CHECK-SD-NEXT:    bic v4.16b, v6.16b, v4.16b
; CHECK-SD-NEXT:    bic v5.16b, v6.16b, v5.16b
; CHECK-SD-NEXT:    neg v6.4s, v7.4s
; CHECK-SD-NEXT:    neg v7.4s, v16.4s
; CHECK-SD-NEXT:    ushl v0.4s, v0.4s, v4.4s
; CHECK-SD-NEXT:    ushl v1.4s, v1.4s, v5.4s
; CHECK-SD-NEXT:    ushl v2.4s, v2.4s, v6.4s
; CHECK-SD-NEXT:    ushl v3.4s, v3.4s, v7.4s
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v8i32:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    movi v6.4s, #31
; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #1
; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #1
; CHECK-GI-NEXT:    and v7.16b, v4.16b, v6.16b
; CHECK-GI-NEXT:    and v16.16b, v5.16b, v6.16b
; CHECK-GI-NEXT:    bic v4.16b, v6.16b, v4.16b
; CHECK-GI-NEXT:    bic v5.16b, v6.16b, v5.16b
; CHECK-GI-NEXT:    neg v6.4s, v7.4s
; CHECK-GI-NEXT:    neg v7.4s, v16.4s
; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v4.4s
; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v5.4s
; CHECK-GI-NEXT:    ushl v2.4s, v2.4s, v6.4s
; CHECK-GI-NEXT:    ushl v3.4s, v3.4s, v7.4s
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i32> @llvm.fshr(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c)
  ret <8 x i32> %d
}

define <2 x i64> @fshl_v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
; CHECK-SD-LABEL: fshl_v2i64:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mov w8, #63 // =0x3f
; CHECK-SD-NEXT:    ushr v1.2d, v1.2d, #1
; CHECK-SD-NEXT:    dup v3.2d, x8
; CHECK-SD-NEXT:    bic v4.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    and v2.16b, v2.16b, v3.16b
; CHECK-SD-NEXT:    neg v3.2d, v4.2d
; CHECK-SD-NEXT:    ushl v0.2d, v0.2d, v2.2d
; CHECK-SD-NEXT:    ushl v1.2d, v1.2d, v3.2d
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v2i64:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    adrp x8, .LCPI86_0
; CHECK-GI-NEXT:    ushr v1.2d, v1.2d, #1
; CHECK-GI-NEXT:    ldr q3, [x8, :lo12:.LCPI86_0]
; CHECK-GI-NEXT:    bic v4.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    and v2.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    neg v3.2d, v4.2d
; CHECK-GI-NEXT:    ushl v0.2d, v0.2d, v2.2d
; CHECK-GI-NEXT:    ushl v1.2d, v1.2d, v3.2d
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i64> @llvm.fshl(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c)
  ret <2 x i64> %d
}

define <2 x i64> @fshr_v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
; CHECK-SD-LABEL: fshr_v2i64:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mov w8, #63 // =0x3f
; CHECK-SD-NEXT:    add v0.2d, v0.2d, v0.2d
; CHECK-SD-NEXT:    dup v3.2d, x8
; CHECK-SD-NEXT:    and v4.16b, v2.16b, v3.16b
; CHECK-SD-NEXT:    bic v2.16b, v3.16b, v2.16b
; CHECK-SD-NEXT:    neg v3.2d, v4.2d
; CHECK-SD-NEXT:    ushl v0.2d, v0.2d, v2.2d
; CHECK-SD-NEXT:    ushl v1.2d, v1.2d, v3.2d
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v2i64:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    adrp x8, .LCPI87_0
; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #1
; CHECK-GI-NEXT:    ldr q3, [x8, :lo12:.LCPI87_0]
; CHECK-GI-NEXT:    and v4.16b, v2.16b, v3.16b
; CHECK-GI-NEXT:    bic v2.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    neg v3.2d, v4.2d
; CHECK-GI-NEXT:    ushl v0.2d, v0.2d, v2.2d
; CHECK-GI-NEXT:    ushl v1.2d, v1.2d, v3.2d
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i64> @llvm.fshr(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c)
  ret <2 x i64> %d
}

define <4 x i64> @fshl_v4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) {
; CHECK-SD-LABEL: fshl_v4i64:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mov w8, #63 // =0x3f
; CHECK-SD-NEXT:    ushr v2.2d, v2.2d, #1
; CHECK-SD-NEXT:    ushr v3.2d, v3.2d, #1
; CHECK-SD-NEXT:    dup v6.2d, x8
; CHECK-SD-NEXT:    bic v7.16b, v6.16b, v4.16b
; CHECK-SD-NEXT:    bic v16.16b, v6.16b, v5.16b
; CHECK-SD-NEXT:    and v4.16b, v4.16b, v6.16b
; CHECK-SD-NEXT:    and v5.16b, v5.16b, v6.16b
; CHECK-SD-NEXT:    neg v6.2d, v7.2d
; CHECK-SD-NEXT:    neg v7.2d, v16.2d
; CHECK-SD-NEXT:    ushl v0.2d, v0.2d, v4.2d
; CHECK-SD-NEXT:    ushl v1.2d, v1.2d, v5.2d
; CHECK-SD-NEXT:    ushl v2.2d, v2.2d, v6.2d
; CHECK-SD-NEXT:    ushl v3.2d, v3.2d, v7.2d
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v4i64:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    adrp x8, .LCPI88_0
; CHECK-GI-NEXT:    ushr v2.2d, v2.2d, #1
; CHECK-GI-NEXT:    ushr v3.2d, v3.2d, #1
; CHECK-GI-NEXT:    ldr q6, [x8, :lo12:.LCPI88_0]
; CHECK-GI-NEXT:    bic v7.16b, v6.16b, v4.16b
; CHECK-GI-NEXT:    bic v16.16b, v6.16b, v5.16b
; CHECK-GI-NEXT:    and v4.16b, v4.16b, v6.16b
; CHECK-GI-NEXT:    and v5.16b, v5.16b, v6.16b
; CHECK-GI-NEXT:    neg v6.2d, v7.2d
; CHECK-GI-NEXT:    neg v7.2d, v16.2d
; CHECK-GI-NEXT:    ushl v0.2d, v0.2d, v4.2d
; CHECK-GI-NEXT:    ushl v1.2d, v1.2d, v5.2d
; CHECK-GI-NEXT:    ushl v2.2d, v2.2d, v6.2d
; CHECK-GI-NEXT:    ushl v3.2d, v3.2d, v7.2d
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i64> @llvm.fshl(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c)
  ret <4 x i64> %d
}

define <4 x i64> @fshr_v4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) {
; CHECK-SD-LABEL: fshr_v4i64:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    mov w8, #63 // =0x3f
; CHECK-SD-NEXT:    add v0.2d, v0.2d, v0.2d
; CHECK-SD-NEXT:    add v1.2d, v1.2d, v1.2d
; CHECK-SD-NEXT:    dup v6.2d, x8
; CHECK-SD-NEXT:    and v7.16b, v4.16b, v6.16b
; CHECK-SD-NEXT:    and v16.16b, v5.16b, v6.16b
; CHECK-SD-NEXT:    bic v4.16b, v6.16b, v4.16b
; CHECK-SD-NEXT:    bic v5.16b, v6.16b, v5.16b
; CHECK-SD-NEXT:    neg v6.2d, v7.2d
; CHECK-SD-NEXT:    neg v7.2d, v16.2d
; CHECK-SD-NEXT:    ushl v0.2d, v0.2d, v4.2d
; CHECK-SD-NEXT:    ushl v1.2d, v1.2d, v5.2d
; CHECK-SD-NEXT:    ushl v2.2d, v2.2d, v6.2d
; CHECK-SD-NEXT:    ushl v3.2d, v3.2d, v7.2d
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v3.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v4i64:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    adrp x8, .LCPI89_0
; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #1
; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #1
; CHECK-GI-NEXT:    ldr q6, [x8, :lo12:.LCPI89_0]
; CHECK-GI-NEXT:    and v7.16b, v4.16b, v6.16b
; CHECK-GI-NEXT:    and v16.16b, v5.16b, v6.16b
; CHECK-GI-NEXT:    bic v4.16b, v6.16b, v4.16b
; CHECK-GI-NEXT:    bic v5.16b, v6.16b, v5.16b
; CHECK-GI-NEXT:    neg v6.2d, v7.2d
; CHECK-GI-NEXT:    neg v7.2d, v16.2d
; CHECK-GI-NEXT:    ushl v0.2d, v0.2d, v4.2d
; CHECK-GI-NEXT:    ushl v1.2d, v1.2d, v5.2d
; CHECK-GI-NEXT:    ushl v2.2d, v2.2d, v6.2d
; CHECK-GI-NEXT:    ushl v3.2d, v3.2d, v7.2d
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v2.16b
; CHECK-GI-NEXT:    orr v1.16b, v1.16b, v3.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i64> @llvm.fshr(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c)
  ret <4 x i64> %d
}

define <2 x i128> @fshl_v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) {
; CHECK-SD-LABEL: fshl_v2i128:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldr x8, [sp]
; CHECK-SD-NEXT:    ldr x10, [sp, #16]
; CHECK-SD-NEXT:    tst x8, #0x40
; CHECK-SD-NEXT:    mvn w13, w8
; CHECK-SD-NEXT:    mvn w16, w10
; CHECK-SD-NEXT:    csel x9, x5, x0, ne
; CHECK-SD-NEXT:    csel x11, x4, x5, ne
; CHECK-SD-NEXT:    csel x14, x0, x1, ne
; CHECK-SD-NEXT:    lsl x12, x9, x8
; CHECK-SD-NEXT:    lsr x11, x11, #1
; CHECK-SD-NEXT:    lsr x9, x9, #1
; CHECK-SD-NEXT:    tst x10, #0x40
; CHECK-SD-NEXT:    lsl x8, x14, x8
; CHECK-SD-NEXT:    csel x14, x7, x2, ne
; CHECK-SD-NEXT:    csel x15, x6, x7, ne
; CHECK-SD-NEXT:    lsr x11, x11, x13
; CHECK-SD-NEXT:    lsr x9, x9, x13
; CHECK-SD-NEXT:    lsr x13, x15, #1
; CHECK-SD-NEXT:    lsr x15, x14, #1
; CHECK-SD-NEXT:    csel x17, x2, x3, ne
; CHECK-SD-NEXT:    lsl x14, x14, x10
; CHECK-SD-NEXT:    orr x0, x12, x11
; CHECK-SD-NEXT:    lsr x13, x13, x16
; CHECK-SD-NEXT:    lsl x10, x17, x10
; CHECK-SD-NEXT:    lsr x15, x15, x16
; CHECK-SD-NEXT:    orr x1, x8, x9
; CHECK-SD-NEXT:    orr x2, x14, x13
; CHECK-SD-NEXT:    orr x3, x10, x15
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v2i128:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    str x19, [sp, #-16]! // 8-byte Folded Spill
; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
; CHECK-GI-NEXT:    .cfi_offset w19, -16
; CHECK-GI-NEXT:    ldr x11, [sp, #16]
; CHECK-GI-NEXT:    mov w10, #64 // =0x40
; CHECK-GI-NEXT:    ldr x12, [sp, #32]
; CHECK-GI-NEXT:    mov w13, #127 // =0x7f
; CHECK-GI-NEXT:    and x9, x11, #0x7f
; CHECK-GI-NEXT:    and x14, x12, #0x7f
; CHECK-GI-NEXT:    mvn x15, x11
; CHECK-GI-NEXT:    sub x8, x10, x9
; CHECK-GI-NEXT:    sub x16, x9, #64
; CHECK-GI-NEXT:    lsl x19, x1, x9
; CHECK-GI-NEXT:    lsr x18, x0, x8
; CHECK-GI-NEXT:    lsl x17, x0, x9
; CHECK-GI-NEXT:    lsl x16, x0, x16
; CHECK-GI-NEXT:    cmp x9, #64
; CHECK-GI-NEXT:    bic x0, x13, x11
; CHECK-GI-NEXT:    mvn x8, x12
; CHECK-GI-NEXT:    orr x18, x18, x19
; CHECK-GI-NEXT:    csel x9, x17, xzr, lo
; CHECK-GI-NEXT:    sub x17, x14, #64
; CHECK-GI-NEXT:    csel x16, x18, x16, lo
; CHECK-GI-NEXT:    tst x11, #0x7f
; CHECK-GI-NEXT:    sub x11, x10, x14
; CHECK-GI-NEXT:    lsr x11, x2, x11
; CHECK-GI-NEXT:    lsl x18, x3, x14
; CHECK-GI-NEXT:    csel x16, x1, x16, eq
; CHECK-GI-NEXT:    lsl x1, x2, x14
; CHECK-GI-NEXT:    lsl x17, x2, x17
; CHECK-GI-NEXT:    cmp x14, #64
; CHECK-GI-NEXT:    lsl x14, x5, #63
; CHECK-GI-NEXT:    orr x11, x11, x18
; CHECK-GI-NEXT:    bic x13, x13, x12
; CHECK-GI-NEXT:    csel x18, x1, xzr, lo
; CHECK-GI-NEXT:    csel x11, x11, x17, lo
; CHECK-GI-NEXT:    tst x12, #0x7f
; CHECK-GI-NEXT:    lsr x12, x5, #1
; CHECK-GI-NEXT:    orr x14, x14, x4, lsr #1
; CHECK-GI-NEXT:    lsl x17, x7, #63
; CHECK-GI-NEXT:    sub x1, x10, x0
; CHECK-GI-NEXT:    csel x11, x3, x11, eq
; CHECK-GI-NEXT:    sub x2, x0, #64
; CHECK-GI-NEXT:    lsr x3, x14, x0
; CHECK-GI-NEXT:    lsl x1, x12, x1
; CHECK-GI-NEXT:    lsr x4, x7, #1
; CHECK-GI-NEXT:    orr x17, x17, x6, lsr #1
; CHECK-GI-NEXT:    lsr x2, x12, x2
; CHECK-GI-NEXT:    cmp x0, #64
; CHECK-GI-NEXT:    orr x1, x3, x1
; CHECK-GI-NEXT:    sub x10, x10, x13
; CHECK-GI-NEXT:    lsr x12, x12, x0
; CHECK-GI-NEXT:    csel x1, x1, x2, lo
; CHECK-GI-NEXT:    tst x15, #0x7f
; CHECK-GI-NEXT:    sub x15, x13, #64
; CHECK-GI-NEXT:    lsr x2, x17, x13
; CHECK-GI-NEXT:    lsl x10, x4, x10
; CHECK-GI-NEXT:    csel x14, x14, x1, eq
; CHECK-GI-NEXT:    cmp x0, #64
; CHECK-GI-NEXT:    lsr x15, x4, x15
; CHECK-GI-NEXT:    lsr x0, x4, x13
; CHECK-GI-NEXT:    csel x12, x12, xzr, lo
; CHECK-GI-NEXT:    orr x10, x2, x10
; CHECK-GI-NEXT:    cmp x13, #64
; CHECK-GI-NEXT:    csel x10, x10, x15, lo
; CHECK-GI-NEXT:    tst x8, #0x7f
; CHECK-GI-NEXT:    orr x1, x16, x12
; CHECK-GI-NEXT:    csel x8, x17, x10, eq
; CHECK-GI-NEXT:    cmp x13, #64
; CHECK-GI-NEXT:    csel x10, x0, xzr, lo
; CHECK-GI-NEXT:    orr x0, x9, x14
; CHECK-GI-NEXT:    orr x2, x18, x8
; CHECK-GI-NEXT:    orr x3, x11, x10
; CHECK-GI-NEXT:    ldr x19, [sp], #16 // 8-byte Folded Reload
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i128> @llvm.fshl(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
  ret <2 x i128> %d
}

define <2 x i128> @fshr_v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) {
; CHECK-SD-LABEL: fshr_v2i128:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    ldr x8, [sp]
; CHECK-SD-NEXT:    ldr x10, [sp, #16]
; CHECK-SD-NEXT:    tst x8, #0x40
; CHECK-SD-NEXT:    mvn w13, w8
; CHECK-SD-NEXT:    csel x9, x4, x5, eq
; CHECK-SD-NEXT:    csel x11, x5, x0, eq
; CHECK-SD-NEXT:    csel x14, x0, x1, eq
; CHECK-SD-NEXT:    tst x10, #0x40
; CHECK-SD-NEXT:    lsr x9, x9, x8
; CHECK-SD-NEXT:    lsl x12, x11, #1
; CHECK-SD-NEXT:    lsr x8, x11, x8
; CHECK-SD-NEXT:    lsl x11, x14, #1
; CHECK-SD-NEXT:    csel x14, x7, x2, eq
; CHECK-SD-NEXT:    csel x15, x2, x3, eq
; CHECK-SD-NEXT:    csel x16, x6, x7, eq
; CHECK-SD-NEXT:    lsl x17, x14, #1
; CHECK-SD-NEXT:    lsl x15, x15, #1
; CHECK-SD-NEXT:    lsl x12, x12, x13
; CHECK-SD-NEXT:    lsl x11, x11, x13
; CHECK-SD-NEXT:    lsr x13, x16, x10
; CHECK-SD-NEXT:    mvn w16, w10
; CHECK-SD-NEXT:    lsr x10, x14, x10
; CHECK-SD-NEXT:    lsl x17, x17, x16
; CHECK-SD-NEXT:    lsl x14, x15, x16
; CHECK-SD-NEXT:    orr x0, x12, x9
; CHECK-SD-NEXT:    orr x1, x11, x8
; CHECK-SD-NEXT:    orr x2, x17, x13
; CHECK-SD-NEXT:    orr x3, x14, x10
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v2i128:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ldr x9, [sp]
; CHECK-GI-NEXT:    lsl x12, x1, #1
; CHECK-GI-NEXT:    mov w11, #127 // =0x7f
; CHECK-GI-NEXT:    mov w14, #64 // =0x40
; CHECK-GI-NEXT:    lsl x15, x0, #1
; CHECK-GI-NEXT:    ldr x8, [sp, #16]
; CHECK-GI-NEXT:    bic x13, x11, x9
; CHECK-GI-NEXT:    orr x12, x12, x0, lsr #63
; CHECK-GI-NEXT:    lsl x1, x3, #1
; CHECK-GI-NEXT:    sub x17, x14, x13
; CHECK-GI-NEXT:    sub x18, x13, #64
; CHECK-GI-NEXT:    lsl x3, x15, x13
; CHECK-GI-NEXT:    lsr x17, x15, x17
; CHECK-GI-NEXT:    lsl x0, x12, x13
; CHECK-GI-NEXT:    lsl x15, x15, x18
; CHECK-GI-NEXT:    bic x11, x11, x8
; CHECK-GI-NEXT:    lsl x18, x2, #1
; CHECK-GI-NEXT:    cmp x13, #64
; CHECK-GI-NEXT:    orr x17, x17, x0
; CHECK-GI-NEXT:    orr x13, x1, x2, lsr #63
; CHECK-GI-NEXT:    mvn x16, x9
; CHECK-GI-NEXT:    csel x15, x17, x15, lo
; CHECK-GI-NEXT:    sub x17, x14, x11
; CHECK-GI-NEXT:    csel x0, x3, xzr, lo
; CHECK-GI-NEXT:    tst x16, #0x7f
; CHECK-GI-NEXT:    sub x16, x11, #64
; CHECK-GI-NEXT:    lsr x17, x18, x17
; CHECK-GI-NEXT:    lsl x2, x13, x11
; CHECK-GI-NEXT:    lsl x1, x18, x11
; CHECK-GI-NEXT:    csel x12, x12, x15, eq
; CHECK-GI-NEXT:    lsl x15, x18, x16
; CHECK-GI-NEXT:    and x10, x9, #0x7f
; CHECK-GI-NEXT:    cmp x11, #64
; CHECK-GI-NEXT:    mvn x11, x8
; CHECK-GI-NEXT:    orr x16, x17, x2
; CHECK-GI-NEXT:    csel x17, x1, xzr, lo
; CHECK-GI-NEXT:    csel x15, x16, x15, lo
; CHECK-GI-NEXT:    tst x11, #0x7f
; CHECK-GI-NEXT:    sub x11, x14, x10
; CHECK-GI-NEXT:    sub x16, x10, #64
; CHECK-GI-NEXT:    lsr x18, x4, x10
; CHECK-GI-NEXT:    lsl x11, x5, x11
; CHECK-GI-NEXT:    csel x13, x13, x15, eq
; CHECK-GI-NEXT:    lsr x15, x5, x16
; CHECK-GI-NEXT:    and x1, x8, #0x7f
; CHECK-GI-NEXT:    orr x11, x18, x11
; CHECK-GI-NEXT:    cmp x10, #64
; CHECK-GI-NEXT:    lsr x16, x5, x10
; CHECK-GI-NEXT:    csel x11, x11, x15, lo
; CHECK-GI-NEXT:    tst x9, #0x7f
; CHECK-GI-NEXT:    sub x9, x14, x1
; CHECK-GI-NEXT:    sub x14, x1, #64
; CHECK-GI-NEXT:    lsr x15, x6, x1
; CHECK-GI-NEXT:    lsl x9, x7, x9
; CHECK-GI-NEXT:    csel x11, x4, x11, eq
; CHECK-GI-NEXT:    cmp x10, #64
; CHECK-GI-NEXT:    lsr x10, x7, x14
; CHECK-GI-NEXT:    csel x14, x16, xzr, lo
; CHECK-GI-NEXT:    orr x9, x15, x9
; CHECK-GI-NEXT:    cmp x1, #64
; CHECK-GI-NEXT:    lsr x15, x7, x1
; CHECK-GI-NEXT:    csel x9, x9, x10, lo
; CHECK-GI-NEXT:    tst x8, #0x7f
; CHECK-GI-NEXT:    csel x8, x6, x9, eq
; CHECK-GI-NEXT:    cmp x1, #64
; CHECK-GI-NEXT:    orr x0, x0, x11
; CHECK-GI-NEXT:    csel x9, x15, xzr, lo
; CHECK-GI-NEXT:    orr x1, x12, x14
; CHECK-GI-NEXT:    orr x2, x17, x8
; CHECK-GI-NEXT:    orr x3, x13, x9
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i128> @llvm.fshr(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c)
  ret <2 x i128> %d
}

define <8 x i8> @rotl_v8i8_c(<8 x i8> %a) {
; CHECK-SD-LABEL: rotl_v8i8_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.8b, v0.8b, #3
; CHECK-SD-NEXT:    usra v1.8b, v0.8b, #5
; CHECK-SD-NEXT:    fmov d0, d1
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v8i8_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v1.8b, v0.8b, #3
; CHECK-GI-NEXT:    ushr v0.8b, v0.8b, #5
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i8> @llvm.fshl(<8 x i8> %a, <8 x i8> %a, <8 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
  ret <8 x i8> %d
}

define <8 x i8> @rotr_v8i8_c(<8 x i8> %a) {
; CHECK-SD-LABEL: rotr_v8i8_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.8b, v0.8b, #5
; CHECK-SD-NEXT:    usra v1.8b, v0.8b, #3
; CHECK-SD-NEXT:    fmov d0, d1
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v8i8_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushr v1.8b, v0.8b, #3
; CHECK-GI-NEXT:    shl v0.8b, v0.8b, #5
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i8> @llvm.fshr(<8 x i8> %a, <8 x i8> %a, <8 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
  ret <8 x i8> %d
}

define <16 x i8> @rotl_v16i8_c(<16 x i8> %a) {
; CHECK-SD-LABEL: rotl_v16i8_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.16b, v0.16b, #3
; CHECK-SD-NEXT:    usra v1.16b, v0.16b, #5
; CHECK-SD-NEXT:    mov v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v16i8_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v1.16b, v0.16b, #3
; CHECK-GI-NEXT:    ushr v0.16b, v0.16b, #5
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <16 x i8> @llvm.fshl(<16 x i8> %a, <16 x i8> %a, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
  ret <16 x i8> %d
}

define <16 x i8> @rotr_v16i8_c(<16 x i8> %a) {
; CHECK-SD-LABEL: rotr_v16i8_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.16b, v0.16b, #5
; CHECK-SD-NEXT:    usra v1.16b, v0.16b, #3
; CHECK-SD-NEXT:    mov v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v16i8_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushr v1.16b, v0.16b, #3
; CHECK-GI-NEXT:    shl v0.16b, v0.16b, #5
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <16 x i8> @llvm.fshr(<16 x i8> %a, <16 x i8> %a, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
  ret <16 x i8> %d
}

define <4 x i16> @rotl_v4i16_c(<4 x i16> %a) {
; CHECK-SD-LABEL: rotl_v4i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.4h, v0.4h, #3
; CHECK-SD-NEXT:    usra v1.4h, v0.4h, #13
; CHECK-SD-NEXT:    fmov d0, d1
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v4i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v1.4h, v0.4h, #3
; CHECK-GI-NEXT:    ushr v0.4h, v0.4h, #13
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i16> @llvm.fshl(<4 x i16> %a, <4 x i16> %a, <4 x i16> <i16 3, i16 3, i16 3, i16 3>)
  ret <4 x i16> %d
}

define <4 x i16> @rotr_v4i16_c(<4 x i16> %a) {
; CHECK-SD-LABEL: rotr_v4i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.4h, v0.4h, #13
; CHECK-SD-NEXT:    usra v1.4h, v0.4h, #3
; CHECK-SD-NEXT:    fmov d0, d1
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v4i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushr v1.4h, v0.4h, #3
; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #13
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i16> @llvm.fshr(<4 x i16> %a, <4 x i16> %a, <4 x i16> <i16 3, i16 3, i16 3, i16 3>)
  ret <4 x i16> %d
}

define <7 x i16> @rotl_v7i16_c(<7 x i16> %a) {
; CHECK-SD-LABEL: rotl_v7i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    adrp x8, .LCPI98_0
; CHECK-SD-NEXT:    adrp x9, .LCPI98_1
; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI98_0]
; CHECK-SD-NEXT:    ldr q2, [x9, :lo12:.LCPI98_1]
; CHECK-SD-NEXT:    ushl v1.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v7i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #13 // =0xd
; CHECK-GI-NEXT:    mov w9, #3 // =0x3
; CHECK-GI-NEXT:    fmov s1, w8
; CHECK-GI-NEXT:    fmov s2, w9
; CHECK-GI-NEXT:    mov v1.h[1], w8
; CHECK-GI-NEXT:    mov v2.h[1], w9
; CHECK-GI-NEXT:    mov v1.h[2], w8
; CHECK-GI-NEXT:    mov v2.h[2], w9
; CHECK-GI-NEXT:    mov v1.h[3], w8
; CHECK-GI-NEXT:    mov v2.h[3], w9
; CHECK-GI-NEXT:    mov v1.h[4], w8
; CHECK-GI-NEXT:    mov v2.h[4], w9
; CHECK-GI-NEXT:    mov v1.h[5], w8
; CHECK-GI-NEXT:    mov v2.h[5], w9
; CHECK-GI-NEXT:    mov v1.h[6], w8
; CHECK-GI-NEXT:    mov v2.h[6], w9
; CHECK-GI-NEXT:    neg v1.8h, v1.8h
; CHECK-GI-NEXT:    ushl v2.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    ushl v0.8h, v0.8h, v1.8h
; CHECK-GI-NEXT:    orr v0.16b, v2.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i16> @llvm.fshl(<7 x i16> %a, <7 x i16> %a, <7 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
  ret <7 x i16> %d
}

define <7 x i16> @rotr_v7i16_c(<7 x i16> %a) {
; CHECK-SD-LABEL: rotr_v7i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    adrp x8, .LCPI99_0
; CHECK-SD-NEXT:    adrp x9, .LCPI99_1
; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI99_0]
; CHECK-SD-NEXT:    ldr q2, [x9, :lo12:.LCPI99_1]
; CHECK-SD-NEXT:    ushl v1.8h, v0.8h, v1.8h
; CHECK-SD-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v7i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #3 // =0x3
; CHECK-GI-NEXT:    mov w9, #13 // =0xd
; CHECK-GI-NEXT:    fmov s1, w8
; CHECK-GI-NEXT:    fmov s2, w9
; CHECK-GI-NEXT:    mov v1.h[1], w8
; CHECK-GI-NEXT:    mov v2.h[1], w9
; CHECK-GI-NEXT:    mov v1.h[2], w8
; CHECK-GI-NEXT:    mov v2.h[2], w9
; CHECK-GI-NEXT:    mov v1.h[3], w8
; CHECK-GI-NEXT:    mov v2.h[3], w9
; CHECK-GI-NEXT:    mov v1.h[4], w8
; CHECK-GI-NEXT:    mov v2.h[4], w9
; CHECK-GI-NEXT:    mov v1.h[5], w8
; CHECK-GI-NEXT:    mov v2.h[5], w9
; CHECK-GI-NEXT:    mov v1.h[6], w8
; CHECK-GI-NEXT:    mov v2.h[6], w9
; CHECK-GI-NEXT:    neg v1.8h, v1.8h
; CHECK-GI-NEXT:    ushl v1.8h, v0.8h, v1.8h
; CHECK-GI-NEXT:    ushl v0.8h, v0.8h, v2.8h
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i16> @llvm.fshr(<7 x i16> %a, <7 x i16> %a, <7 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
  ret <7 x i16> %d
}

define <8 x i16> @rotl_v8i16_c(<8 x i16> %a) {
; CHECK-SD-LABEL: rotl_v8i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.8h, v0.8h, #3
; CHECK-SD-NEXT:    usra v1.8h, v0.8h, #13
; CHECK-SD-NEXT:    mov v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v8i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v1.8h, v0.8h, #3
; CHECK-GI-NEXT:    ushr v0.8h, v0.8h, #13
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i16> @llvm.fshl(<8 x i16> %a, <8 x i16> %a, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
  ret <8 x i16> %d
}

define <8 x i16> @rotr_v8i16_c(<8 x i16> %a) {
; CHECK-SD-LABEL: rotr_v8i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.8h, v0.8h, #13
; CHECK-SD-NEXT:    usra v1.8h, v0.8h, #3
; CHECK-SD-NEXT:    mov v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v8i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushr v1.8h, v0.8h, #3
; CHECK-GI-NEXT:    shl v0.8h, v0.8h, #13
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i16> @llvm.fshr(<8 x i16> %a, <8 x i16> %a, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
  ret <8 x i16> %d
}

define <16 x i16> @rotl_v16i16_c(<16 x i16> %a) {
; CHECK-SD-LABEL: rotl_v16i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v2.8h, v0.8h, #3
; CHECK-SD-NEXT:    shl v3.8h, v1.8h, #3
; CHECK-SD-NEXT:    usra v2.8h, v0.8h, #13
; CHECK-SD-NEXT:    usra v3.8h, v1.8h, #13
; CHECK-SD-NEXT:    mov v0.16b, v2.16b
; CHECK-SD-NEXT:    mov v1.16b, v3.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v16i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v2.8h, v0.8h, #3
; CHECK-GI-NEXT:    shl v3.8h, v1.8h, #3
; CHECK-GI-NEXT:    ushr v0.8h, v0.8h, #13
; CHECK-GI-NEXT:    ushr v1.8h, v1.8h, #13
; CHECK-GI-NEXT:    orr v0.16b, v2.16b, v0.16b
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <16 x i16> @llvm.fshl(<16 x i16> %a, <16 x i16> %a, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
  ret <16 x i16> %d
}

define <16 x i16> @rotr_v16i16_c(<16 x i16> %a) {
; CHECK-SD-LABEL: rotr_v16i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v2.8h, v0.8h, #13
; CHECK-SD-NEXT:    shl v3.8h, v1.8h, #13
; CHECK-SD-NEXT:    usra v2.8h, v0.8h, #3
; CHECK-SD-NEXT:    usra v3.8h, v1.8h, #3
; CHECK-SD-NEXT:    mov v0.16b, v2.16b
; CHECK-SD-NEXT:    mov v1.16b, v3.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v16i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushr v2.8h, v0.8h, #3
; CHECK-GI-NEXT:    ushr v3.8h, v1.8h, #3
; CHECK-GI-NEXT:    shl v0.8h, v0.8h, #13
; CHECK-GI-NEXT:    shl v1.8h, v1.8h, #13
; CHECK-GI-NEXT:    orr v0.16b, v2.16b, v0.16b
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <16 x i16> @llvm.fshr(<16 x i16> %a, <16 x i16> %a, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
  ret <16 x i16> %d
}

define <2 x i32> @rotl_v2i32_c(<2 x i32> %a) {
; CHECK-SD-LABEL: rotl_v2i32_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.2s, v0.2s, #3
; CHECK-SD-NEXT:    usra v1.2s, v0.2s, #29
; CHECK-SD-NEXT:    fmov d0, d1
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v2i32_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v1.2s, v0.2s, #3
; CHECK-GI-NEXT:    ushr v0.2s, v0.2s, #29
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i32> @llvm.fshl(<2 x i32> %a, <2 x i32> %a, <2 x i32> <i32 3, i32 3>)
  ret <2 x i32> %d
}

define <2 x i32> @rotr_v2i32_c(<2 x i32> %a) {
; CHECK-SD-LABEL: rotr_v2i32_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.2s, v0.2s, #29
; CHECK-SD-NEXT:    usra v1.2s, v0.2s, #3
; CHECK-SD-NEXT:    fmov d0, d1
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v2i32_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushr v1.2s, v0.2s, #3
; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #29
; CHECK-GI-NEXT:    orr v0.8b, v1.8b, v0.8b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i32> @llvm.fshr(<2 x i32> %a, <2 x i32> %a, <2 x i32> <i32 3, i32 3>)
  ret <2 x i32> %d
}

define <4 x i32> @rotl_v4i32_c(<4 x i32> %a) {
; CHECK-SD-LABEL: rotl_v4i32_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.4s, v0.4s, #3
; CHECK-SD-NEXT:    usra v1.4s, v0.4s, #29
; CHECK-SD-NEXT:    mov v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v4i32_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v1.4s, v0.4s, #3
; CHECK-GI-NEXT:    ushr v0.4s, v0.4s, #29
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i32> @llvm.fshl(<4 x i32> %a, <4 x i32> %a, <4 x i32> <i32 3, i32 3, i32 3, i32 3>)
  ret <4 x i32> %d
}

define <4 x i32> @rotr_v4i32_c(<4 x i32> %a) {
; CHECK-SD-LABEL: rotr_v4i32_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.4s, v0.4s, #29
; CHECK-SD-NEXT:    usra v1.4s, v0.4s, #3
; CHECK-SD-NEXT:    mov v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v4i32_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushr v1.4s, v0.4s, #3
; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #29
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i32> @llvm.fshr(<4 x i32> %a, <4 x i32> %a, <4 x i32> <i32 3, i32 3, i32 3, i32 3>)
  ret <4 x i32> %d
}

define <7 x i32> @rotl_v7i32_c(<7 x i32> %a) {
; CHECK-SD-LABEL: rotl_v7i32_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    fmov s0, w0
; CHECK-SD-NEXT:    fmov s1, w4
; CHECK-SD-NEXT:    adrp x8, .LCPI108_0
; CHECK-SD-NEXT:    adrp x9, .LCPI108_1
; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI108_0]
; CHECK-SD-NEXT:    ldr q3, [x9, :lo12:.LCPI108_1]
; CHECK-SD-NEXT:    mov v0.s[1], w1
; CHECK-SD-NEXT:    mov v1.s[1], w5
; CHECK-SD-NEXT:    mov v0.s[2], w2
; CHECK-SD-NEXT:    mov v1.s[2], w6
; CHECK-SD-NEXT:    mov v0.s[3], w3
; CHECK-SD-NEXT:    ushl v2.4s, v1.4s, v2.4s
; CHECK-SD-NEXT:    ushl v1.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    shl v4.4s, v0.4s, #3
; CHECK-SD-NEXT:    usra v4.4s, v0.4s, #29
; CHECK-SD-NEXT:    orr v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    mov w1, v4.s[1]
; CHECK-SD-NEXT:    mov w2, v4.s[2]
; CHECK-SD-NEXT:    mov w3, v4.s[3]
; CHECK-SD-NEXT:    mov w5, v0.s[1]
; CHECK-SD-NEXT:    mov w6, v0.s[2]
; CHECK-SD-NEXT:    fmov w0, s4
; CHECK-SD-NEXT:    fmov w4, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v7i32_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov v0.s[0], w0
; CHECK-GI-NEXT:    mov v1.s[0], w0
; CHECK-GI-NEXT:    mov w8, #29 // =0x1d
; CHECK-GI-NEXT:    mov v2.s[0], w8
; CHECK-GI-NEXT:    mov w9, #3 // =0x3
; CHECK-GI-NEXT:    mov v3.s[0], w4
; CHECK-GI-NEXT:    mov v4.s[0], w9
; CHECK-GI-NEXT:    mov v5.s[0], w4
; CHECK-GI-NEXT:    mov v0.s[1], w1
; CHECK-GI-NEXT:    mov v1.s[1], w1
; CHECK-GI-NEXT:    mov v2.s[1], w8
; CHECK-GI-NEXT:    mov v3.s[1], w5
; CHECK-GI-NEXT:    mov v4.s[1], w9
; CHECK-GI-NEXT:    mov v5.s[1], w5
; CHECK-GI-NEXT:    mov v0.s[2], w2
; CHECK-GI-NEXT:    mov v1.s[2], w2
; CHECK-GI-NEXT:    mov v2.s[2], w8
; CHECK-GI-NEXT:    mov v3.s[2], w6
; CHECK-GI-NEXT:    mov v4.s[2], w9
; CHECK-GI-NEXT:    mov v5.s[2], w6
; CHECK-GI-NEXT:    mov v0.s[3], w3
; CHECK-GI-NEXT:    mov v1.s[3], w3
; CHECK-GI-NEXT:    neg v2.4s, v2.4s
; CHECK-GI-NEXT:    ushl v3.4s, v3.4s, v4.4s
; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #3
; CHECK-GI-NEXT:    ushr v1.4s, v1.4s, #29
; CHECK-GI-NEXT:    ushl v2.4s, v5.4s, v2.4s
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v2.16b
; CHECK-GI-NEXT:    mov s2, v0.s[1]
; CHECK-GI-NEXT:    mov s3, v0.s[2]
; CHECK-GI-NEXT:    mov s4, v0.s[3]
; CHECK-GI-NEXT:    mov s5, v1.s[1]
; CHECK-GI-NEXT:    mov s6, v1.s[2]
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    fmov w4, s1
; CHECK-GI-NEXT:    fmov w1, s2
; CHECK-GI-NEXT:    fmov w2, s3
; CHECK-GI-NEXT:    fmov w3, s4
; CHECK-GI-NEXT:    fmov w5, s5
; CHECK-GI-NEXT:    fmov w6, s6
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i32> @llvm.fshl(<7 x i32> %a, <7 x i32> %a, <7 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>)
  ret <7 x i32> %d
}

define <7 x i32> @rotr_v7i32_c(<7 x i32> %a) {
; CHECK-SD-LABEL: rotr_v7i32_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    fmov s0, w0
; CHECK-SD-NEXT:    fmov s1, w4
; CHECK-SD-NEXT:    adrp x8, .LCPI109_0
; CHECK-SD-NEXT:    adrp x9, .LCPI109_1
; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI109_0]
; CHECK-SD-NEXT:    ldr q3, [x9, :lo12:.LCPI109_1]
; CHECK-SD-NEXT:    mov v0.s[1], w1
; CHECK-SD-NEXT:    mov v1.s[1], w5
; CHECK-SD-NEXT:    mov v0.s[2], w2
; CHECK-SD-NEXT:    mov v1.s[2], w6
; CHECK-SD-NEXT:    mov v0.s[3], w3
; CHECK-SD-NEXT:    ushl v2.4s, v1.4s, v2.4s
; CHECK-SD-NEXT:    ushl v1.4s, v1.4s, v3.4s
; CHECK-SD-NEXT:    shl v4.4s, v0.4s, #29
; CHECK-SD-NEXT:    usra v4.4s, v0.4s, #3
; CHECK-SD-NEXT:    orr v0.16b, v1.16b, v2.16b
; CHECK-SD-NEXT:    mov w1, v4.s[1]
; CHECK-SD-NEXT:    mov w2, v4.s[2]
; CHECK-SD-NEXT:    mov w3, v4.s[3]
; CHECK-SD-NEXT:    mov w5, v0.s[1]
; CHECK-SD-NEXT:    mov w6, v0.s[2]
; CHECK-SD-NEXT:    fmov w0, s4
; CHECK-SD-NEXT:    fmov w4, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v7i32_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov v0.s[0], w0
; CHECK-GI-NEXT:    mov v1.s[0], w0
; CHECK-GI-NEXT:    mov w8, #3 // =0x3
; CHECK-GI-NEXT:    mov v2.s[0], w8
; CHECK-GI-NEXT:    mov w9, #29 // =0x1d
; CHECK-GI-NEXT:    mov v3.s[0], w4
; CHECK-GI-NEXT:    mov v4.s[0], w4
; CHECK-GI-NEXT:    mov v5.s[0], w9
; CHECK-GI-NEXT:    mov v0.s[1], w1
; CHECK-GI-NEXT:    mov v1.s[1], w1
; CHECK-GI-NEXT:    mov v2.s[1], w8
; CHECK-GI-NEXT:    mov v3.s[1], w5
; CHECK-GI-NEXT:    mov v4.s[1], w5
; CHECK-GI-NEXT:    mov v5.s[1], w9
; CHECK-GI-NEXT:    mov v0.s[2], w2
; CHECK-GI-NEXT:    mov v1.s[2], w2
; CHECK-GI-NEXT:    mov v2.s[2], w8
; CHECK-GI-NEXT:    mov v3.s[2], w6
; CHECK-GI-NEXT:    mov v4.s[2], w6
; CHECK-GI-NEXT:    mov v5.s[2], w9
; CHECK-GI-NEXT:    mov v0.s[3], w3
; CHECK-GI-NEXT:    mov v1.s[3], w3
; CHECK-GI-NEXT:    neg v2.4s, v2.4s
; CHECK-GI-NEXT:    ushl v4.4s, v4.4s, v5.4s
; CHECK-GI-NEXT:    ushr v0.4s, v0.4s, #3
; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #29
; CHECK-GI-NEXT:    ushl v2.4s, v3.4s, v2.4s
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    orr v1.16b, v2.16b, v4.16b
; CHECK-GI-NEXT:    mov s2, v0.s[1]
; CHECK-GI-NEXT:    mov s3, v0.s[2]
; CHECK-GI-NEXT:    mov s4, v0.s[3]
; CHECK-GI-NEXT:    mov s5, v1.s[1]
; CHECK-GI-NEXT:    mov s6, v1.s[2]
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    fmov w4, s1
; CHECK-GI-NEXT:    fmov w1, s2
; CHECK-GI-NEXT:    fmov w2, s3
; CHECK-GI-NEXT:    fmov w3, s4
; CHECK-GI-NEXT:    fmov w5, s5
; CHECK-GI-NEXT:    fmov w6, s6
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i32> @llvm.fshr(<7 x i32> %a, <7 x i32> %a, <7 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>)
  ret <7 x i32> %d
}

define <8 x i32> @rotl_v8i32_c(<8 x i32> %a) {
; CHECK-SD-LABEL: rotl_v8i32_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v2.4s, v0.4s, #3
; CHECK-SD-NEXT:    shl v3.4s, v1.4s, #3
; CHECK-SD-NEXT:    usra v2.4s, v0.4s, #29
; CHECK-SD-NEXT:    usra v3.4s, v1.4s, #29
; CHECK-SD-NEXT:    mov v0.16b, v2.16b
; CHECK-SD-NEXT:    mov v1.16b, v3.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v8i32_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v2.4s, v0.4s, #3
; CHECK-GI-NEXT:    shl v3.4s, v1.4s, #3
; CHECK-GI-NEXT:    ushr v0.4s, v0.4s, #29
; CHECK-GI-NEXT:    ushr v1.4s, v1.4s, #29
; CHECK-GI-NEXT:    orr v0.16b, v2.16b, v0.16b
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i32> @llvm.fshl(<8 x i32> %a, <8 x i32> %a, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>)
  ret <8 x i32> %d
}

define <8 x i32> @rotr_v8i32_c(<8 x i32> %a) {
; CHECK-SD-LABEL: rotr_v8i32_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v2.4s, v0.4s, #29
; CHECK-SD-NEXT:    shl v3.4s, v1.4s, #29
; CHECK-SD-NEXT:    usra v2.4s, v0.4s, #3
; CHECK-SD-NEXT:    usra v3.4s, v1.4s, #3
; CHECK-SD-NEXT:    mov v0.16b, v2.16b
; CHECK-SD-NEXT:    mov v1.16b, v3.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v8i32_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushr v2.4s, v0.4s, #3
; CHECK-GI-NEXT:    ushr v3.4s, v1.4s, #3
; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #29
; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #29
; CHECK-GI-NEXT:    orr v0.16b, v2.16b, v0.16b
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i32> @llvm.fshr(<8 x i32> %a, <8 x i32> %a, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>)
  ret <8 x i32> %d
}

define <2 x i64> @rotl_v2i64_c(<2 x i64> %a) {
; CHECK-SD-LABEL: rotl_v2i64_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.2d, v0.2d, #3
; CHECK-SD-NEXT:    usra v1.2d, v0.2d, #61
; CHECK-SD-NEXT:    mov v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v2i64_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v1.2d, v0.2d, #3
; CHECK-GI-NEXT:    ushr v0.2d, v0.2d, #61
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i64> @llvm.fshl(<2 x i64> %a, <2 x i64> %a, <2 x i64> <i64 3, i64 3>)
  ret <2 x i64> %d
}

define <2 x i64> @rotr_v2i64_c(<2 x i64> %a) {
; CHECK-SD-LABEL: rotr_v2i64_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.2d, v0.2d, #61
; CHECK-SD-NEXT:    usra v1.2d, v0.2d, #3
; CHECK-SD-NEXT:    mov v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v2i64_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushr v1.2d, v0.2d, #3
; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #61
; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i64> @llvm.fshr(<2 x i64> %a, <2 x i64> %a, <2 x i64> <i64 3, i64 3>)
  ret <2 x i64> %d
}

define <4 x i64> @rotl_v4i64_c(<4 x i64> %a) {
; CHECK-SD-LABEL: rotl_v4i64_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v2.2d, v0.2d, #3
; CHECK-SD-NEXT:    shl v3.2d, v1.2d, #3
; CHECK-SD-NEXT:    usra v2.2d, v0.2d, #61
; CHECK-SD-NEXT:    usra v3.2d, v1.2d, #61
; CHECK-SD-NEXT:    mov v0.16b, v2.16b
; CHECK-SD-NEXT:    mov v1.16b, v3.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v4i64_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v2.2d, v0.2d, #3
; CHECK-GI-NEXT:    shl v3.2d, v1.2d, #3
; CHECK-GI-NEXT:    ushr v0.2d, v0.2d, #61
; CHECK-GI-NEXT:    ushr v1.2d, v1.2d, #61
; CHECK-GI-NEXT:    orr v0.16b, v2.16b, v0.16b
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i64> @llvm.fshl(<4 x i64> %a, <4 x i64> %a, <4 x i64> <i64 3, i64 3, i64 3, i64 3>)
  ret <4 x i64> %d
}

define <4 x i64> @rotr_v4i64_c(<4 x i64> %a) {
; CHECK-SD-LABEL: rotr_v4i64_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v2.2d, v0.2d, #61
; CHECK-SD-NEXT:    shl v3.2d, v1.2d, #61
; CHECK-SD-NEXT:    usra v2.2d, v0.2d, #3
; CHECK-SD-NEXT:    usra v3.2d, v1.2d, #3
; CHECK-SD-NEXT:    mov v0.16b, v2.16b
; CHECK-SD-NEXT:    mov v1.16b, v3.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v4i64_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    ushr v2.2d, v0.2d, #3
; CHECK-GI-NEXT:    ushr v3.2d, v1.2d, #3
; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #61
; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #61
; CHECK-GI-NEXT:    orr v0.16b, v2.16b, v0.16b
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i64> @llvm.fshr(<4 x i64> %a, <4 x i64> %a, <4 x i64> <i64 3, i64 3, i64 3, i64 3>)
  ret <4 x i64> %d
}

define <2 x i128> @rotl_v2i128_c(<2 x i128> %a) {
; CHECK-SD-LABEL: rotl_v2i128_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    extr x8, x2, x3, #61
; CHECK-SD-NEXT:    extr x9, x0, x1, #61
; CHECK-SD-NEXT:    extr x1, x1, x0, #61
; CHECK-SD-NEXT:    extr x3, x3, x2, #61
; CHECK-SD-NEXT:    mov x0, x9
; CHECK-SD-NEXT:    mov x2, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotl_v2i128_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    lsr x8, x1, #61
; CHECK-GI-NEXT:    lsl x9, x1, #3
; CHECK-GI-NEXT:    lsl x10, x3, #3
; CHECK-GI-NEXT:    lsr x11, x3, #61
; CHECK-GI-NEXT:    orr x8, x8, x0, lsl #3
; CHECK-GI-NEXT:    orr x1, x9, x0, lsr #61
; CHECK-GI-NEXT:    orr x3, x10, x2, lsr #61
; CHECK-GI-NEXT:    orr x2, x11, x2, lsl #3
; CHECK-GI-NEXT:    mov x0, x8
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i128> @llvm.fshl(<2 x i128> %a, <2 x i128> %a, <2 x i128> <i128 3, i128 3>)
  ret <2 x i128> %d
}

define <2 x i128> @rotr_v2i128_c(<2 x i128> %a) {
; CHECK-SD-LABEL: rotr_v2i128_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    extr x8, x3, x2, #3
; CHECK-SD-NEXT:    extr x9, x1, x0, #3
; CHECK-SD-NEXT:    extr x1, x0, x1, #3
; CHECK-SD-NEXT:    extr x3, x2, x3, #3
; CHECK-SD-NEXT:    mov x0, x9
; CHECK-SD-NEXT:    mov x2, x8
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rotr_v2i128_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    lsl x8, x1, #61
; CHECK-GI-NEXT:    lsl x9, x3, #61
; CHECK-GI-NEXT:    lsl x10, x0, #61
; CHECK-GI-NEXT:    lsl x11, x2, #61
; CHECK-GI-NEXT:    orr x0, x8, x0, lsr #3
; CHECK-GI-NEXT:    orr x2, x9, x2, lsr #3
; CHECK-GI-NEXT:    orr x1, x10, x1, lsr #3
; CHECK-GI-NEXT:    orr x3, x11, x3, lsr #3
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i128> @llvm.fshr(<2 x i128> %a, <2 x i128> %a, <2 x i128> <i128 3, i128 3>)
  ret <2 x i128> %d
}

define <8 x i8> @fshl_v8i8_c(<8 x i8> %a, <8 x i8> %b) {
; CHECK-SD-LABEL: fshl_v8i8_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v0.8b, v0.8b, #3
; CHECK-SD-NEXT:    usra v0.8b, v1.8b, #5
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v8i8_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.8b, v0.8b, #3
; CHECK-GI-NEXT:    usra v0.8b, v1.8b, #5
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i8> @llvm.fshl(<8 x i8> %a, <8 x i8> %b, <8 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
  ret <8 x i8> %d
}

define <8 x i8> @fshr_v8i8_c(<8 x i8> %a, <8 x i8> %b) {
; CHECK-SD-LABEL: fshr_v8i8_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v0.8b, v0.8b, #5
; CHECK-SD-NEXT:    usra v0.8b, v1.8b, #3
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v8i8_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.8b, v0.8b, #5
; CHECK-GI-NEXT:    usra v0.8b, v1.8b, #3
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i8> @llvm.fshr(<8 x i8> %a, <8 x i8> %b, <8 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
  ret <8 x i8> %d
}

define <16 x i8> @fshl_v16i8_c(<16 x i8> %a, <16 x i8> %b) {
; CHECK-SD-LABEL: fshl_v16i8_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v0.16b, v0.16b, #3
; CHECK-SD-NEXT:    usra v0.16b, v1.16b, #5
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v16i8_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.16b, v0.16b, #3
; CHECK-GI-NEXT:    usra v0.16b, v1.16b, #5
; CHECK-GI-NEXT:    ret
entry:
  %d = call <16 x i8> @llvm.fshl(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
  ret <16 x i8> %d
}

define <16 x i8> @fshr_v16i8_c(<16 x i8> %a, <16 x i8> %b) {
; CHECK-SD-LABEL: fshr_v16i8_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v0.16b, v0.16b, #5
; CHECK-SD-NEXT:    usra v0.16b, v1.16b, #3
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v16i8_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.16b, v0.16b, #5
; CHECK-GI-NEXT:    usra v0.16b, v1.16b, #3
; CHECK-GI-NEXT:    ret
entry:
  %d = call <16 x i8> @llvm.fshr(<16 x i8> %a, <16 x i8> %b, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
  ret <16 x i8> %d
}

define <4 x i16> @fshl_v4i16_c(<4 x i16> %a, <4 x i16> %b) {
; CHECK-SD-LABEL: fshl_v4i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #3
; CHECK-SD-NEXT:    usra v0.4h, v1.4h, #13
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v4i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #3
; CHECK-GI-NEXT:    usra v0.4h, v1.4h, #13
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i16> @llvm.fshl(<4 x i16> %a, <4 x i16> %b, <4 x i16> <i16 3, i16 3, i16 3, i16 3>)
  ret <4 x i16> %d
}

define <4 x i16> @fshr_v4i16_c(<4 x i16> %a, <4 x i16> %b) {
; CHECK-SD-LABEL: fshr_v4i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #13
; CHECK-SD-NEXT:    usra v0.4h, v1.4h, #3
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v4i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #13
; CHECK-GI-NEXT:    usra v0.4h, v1.4h, #3
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i16> @llvm.fshr(<4 x i16> %a, <4 x i16> %b, <4 x i16> <i16 3, i16 3, i16 3, i16 3>)
  ret <4 x i16> %d
}

define <7 x i16> @fshl_v7i16_c(<7 x i16> %a, <7 x i16> %b) {
; CHECK-SD-LABEL: fshl_v7i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    adrp x8, .LCPI124_0
; CHECK-SD-NEXT:    adrp x9, .LCPI124_1
; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI124_0]
; CHECK-SD-NEXT:    ldr q3, [x9, :lo12:.LCPI124_1]
; CHECK-SD-NEXT:    ushl v1.8h, v1.8h, v2.8h
; CHECK-SD-NEXT:    ushl v0.8h, v0.8h, v3.8h
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v7i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #13 // =0xd
; CHECK-GI-NEXT:    mov w9, #3 // =0x3
; CHECK-GI-NEXT:    fmov s2, w8
; CHECK-GI-NEXT:    fmov s3, w9
; CHECK-GI-NEXT:    mov v2.h[1], w8
; CHECK-GI-NEXT:    mov v3.h[1], w9
; CHECK-GI-NEXT:    mov v2.h[2], w8
; CHECK-GI-NEXT:    mov v3.h[2], w9
; CHECK-GI-NEXT:    mov v2.h[3], w8
; CHECK-GI-NEXT:    mov v3.h[3], w9
; CHECK-GI-NEXT:    mov v2.h[4], w8
; CHECK-GI-NEXT:    mov v3.h[4], w9
; CHECK-GI-NEXT:    mov v2.h[5], w8
; CHECK-GI-NEXT:    mov v3.h[5], w9
; CHECK-GI-NEXT:    mov v2.h[6], w8
; CHECK-GI-NEXT:    mov v3.h[6], w9
; CHECK-GI-NEXT:    neg v2.8h, v2.8h
; CHECK-GI-NEXT:    ushl v0.8h, v0.8h, v3.8h
; CHECK-GI-NEXT:    ushl v1.8h, v1.8h, v2.8h
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i16> @llvm.fshl(<7 x i16> %a, <7 x i16> %b, <7 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
  ret <7 x i16> %d
}

define <7 x i16> @fshr_v7i16_c(<7 x i16> %a, <7 x i16> %b) {
; CHECK-SD-LABEL: fshr_v7i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    adrp x8, .LCPI125_0
; CHECK-SD-NEXT:    adrp x9, .LCPI125_1
; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI125_0]
; CHECK-SD-NEXT:    ldr q3, [x9, :lo12:.LCPI125_1]
; CHECK-SD-NEXT:    ushl v1.8h, v1.8h, v2.8h
; CHECK-SD-NEXT:    ushl v0.8h, v0.8h, v3.8h
; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v7i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov w8, #3 // =0x3
; CHECK-GI-NEXT:    mov w9, #13 // =0xd
; CHECK-GI-NEXT:    fmov s2, w8
; CHECK-GI-NEXT:    fmov s3, w9
; CHECK-GI-NEXT:    mov v2.h[1], w8
; CHECK-GI-NEXT:    mov v3.h[1], w9
; CHECK-GI-NEXT:    mov v2.h[2], w8
; CHECK-GI-NEXT:    mov v3.h[2], w9
; CHECK-GI-NEXT:    mov v2.h[3], w8
; CHECK-GI-NEXT:    mov v3.h[3], w9
; CHECK-GI-NEXT:    mov v2.h[4], w8
; CHECK-GI-NEXT:    mov v3.h[4], w9
; CHECK-GI-NEXT:    mov v2.h[5], w8
; CHECK-GI-NEXT:    mov v3.h[5], w9
; CHECK-GI-NEXT:    mov v2.h[6], w8
; CHECK-GI-NEXT:    mov v3.h[6], w9
; CHECK-GI-NEXT:    neg v2.8h, v2.8h
; CHECK-GI-NEXT:    ushl v0.8h, v0.8h, v3.8h
; CHECK-GI-NEXT:    ushl v1.8h, v1.8h, v2.8h
; CHECK-GI-NEXT:    orr v0.16b, v0.16b, v1.16b
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i16> @llvm.fshr(<7 x i16> %a, <7 x i16> %b, <7 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
  ret <7 x i16> %d
}

define <8 x i16> @fshl_v8i16_c(<8 x i16> %a, <8 x i16> %b) {
; CHECK-SD-LABEL: fshl_v8i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v0.8h, v0.8h, #3
; CHECK-SD-NEXT:    usra v0.8h, v1.8h, #13
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v8i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.8h, v0.8h, #3
; CHECK-GI-NEXT:    usra v0.8h, v1.8h, #13
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i16> @llvm.fshl(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
  ret <8 x i16> %d
}

define <8 x i16> @fshr_v8i16_c(<8 x i16> %a, <8 x i16> %b) {
; CHECK-SD-LABEL: fshr_v8i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v0.8h, v0.8h, #13
; CHECK-SD-NEXT:    usra v0.8h, v1.8h, #3
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v8i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.8h, v0.8h, #13
; CHECK-GI-NEXT:    usra v0.8h, v1.8h, #3
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i16> @llvm.fshr(<8 x i16> %a, <8 x i16> %b, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
  ret <8 x i16> %d
}

define <16 x i16> @fshl_v16i16_c(<16 x i16> %a, <16 x i16> %b) {
; CHECK-SD-LABEL: fshl_v16i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.8h, v1.8h, #3
; CHECK-SD-NEXT:    shl v0.8h, v0.8h, #3
; CHECK-SD-NEXT:    usra v1.8h, v3.8h, #13
; CHECK-SD-NEXT:    usra v0.8h, v2.8h, #13
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v16i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.8h, v0.8h, #3
; CHECK-GI-NEXT:    shl v1.8h, v1.8h, #3
; CHECK-GI-NEXT:    usra v0.8h, v2.8h, #13
; CHECK-GI-NEXT:    usra v1.8h, v3.8h, #13
; CHECK-GI-NEXT:    ret
entry:
  %d = call <16 x i16> @llvm.fshl(<16 x i16> %a, <16 x i16> %b, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
  ret <16 x i16> %d
}

define <16 x i16> @fshr_v16i16_c(<16 x i16> %a, <16 x i16> %b) {
; CHECK-SD-LABEL: fshr_v16i16_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.8h, v1.8h, #13
; CHECK-SD-NEXT:    shl v0.8h, v0.8h, #13
; CHECK-SD-NEXT:    usra v1.8h, v3.8h, #3
; CHECK-SD-NEXT:    usra v0.8h, v2.8h, #3
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v16i16_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.8h, v0.8h, #13
; CHECK-GI-NEXT:    shl v1.8h, v1.8h, #13
; CHECK-GI-NEXT:    usra v0.8h, v2.8h, #3
; CHECK-GI-NEXT:    usra v1.8h, v3.8h, #3
; CHECK-GI-NEXT:    ret
entry:
  %d = call <16 x i16> @llvm.fshr(<16 x i16> %a, <16 x i16> %b, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
  ret <16 x i16> %d
}

define <2 x i32> @fshl_v2i32_c(<2 x i32> %a, <2 x i32> %b) {
; CHECK-LABEL: fshl_v2i32_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    shl v0.2s, v0.2s, #3
; CHECK-NEXT:    usra v0.2s, v1.2s, #29
; CHECK-NEXT:    ret
entry:
  %d = call <2 x i32> @llvm.fshl(<2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 3, i32 3>)
  ret <2 x i32> %d
}

define <2 x i32> @fshr_v2i32_c(<2 x i32> %a, <2 x i32> %b) {
; CHECK-LABEL: fshr_v2i32_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    shl v0.2s, v0.2s, #29
; CHECK-NEXT:    usra v0.2s, v1.2s, #3
; CHECK-NEXT:    ret
entry:
  %d = call <2 x i32> @llvm.fshr(<2 x i32> %a, <2 x i32> %b, <2 x i32> <i32 3, i32 3>)
  ret <2 x i32> %d
}

define <4 x i32> @fshl_v4i32_c(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: fshl_v4i32_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    shl v0.4s, v0.4s, #3
; CHECK-NEXT:    usra v0.4s, v1.4s, #29
; CHECK-NEXT:    ret
entry:
  %d = call <4 x i32> @llvm.fshl(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 3, i32 3, i32 3, i32 3>)
  ret <4 x i32> %d
}

define <4 x i32> @fshr_v4i32_c(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: fshr_v4i32_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    shl v0.4s, v0.4s, #29
; CHECK-NEXT:    usra v0.4s, v1.4s, #3
; CHECK-NEXT:    ret
entry:
  %d = call <4 x i32> @llvm.fshr(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 3, i32 3, i32 3, i32 3>)
  ret <4 x i32> %d
}

define <7 x i32> @fshl_v7i32_c(<7 x i32> %a, <7 x i32> %b) {
; CHECK-SD-LABEL: fshl_v7i32_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    fmov s0, w0
; CHECK-SD-NEXT:    fmov s2, w4
; CHECK-SD-NEXT:    ldr s1, [sp, #24]
; CHECK-SD-NEXT:    fmov s3, w7
; CHECK-SD-NEXT:    mov x8, sp
; CHECK-SD-NEXT:    add x9, sp, #32
; CHECK-SD-NEXT:    ld1 { v1.s }[1], [x9]
; CHECK-SD-NEXT:    add x9, sp, #40
; CHECK-SD-NEXT:    adrp x10, .LCPI134_1
; CHECK-SD-NEXT:    mov v0.s[1], w1
; CHECK-SD-NEXT:    mov v2.s[1], w5
; CHECK-SD-NEXT:    ldr q5, [x10, :lo12:.LCPI134_1]
; CHECK-SD-NEXT:    ld1 { v3.s }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #8
; CHECK-SD-NEXT:    ld1 { v1.s }[2], [x9]
; CHECK-SD-NEXT:    add x9, sp, #16
; CHECK-SD-NEXT:    mov v0.s[2], w2
; CHECK-SD-NEXT:    mov v2.s[2], w6
; CHECK-SD-NEXT:    ld1 { v3.s }[2], [x8]
; CHECK-SD-NEXT:    adrp x8, .LCPI134_0
; CHECK-SD-NEXT:    ldr q4, [x8, :lo12:.LCPI134_0]
; CHECK-SD-NEXT:    ld1 { v3.s }[3], [x9]
; CHECK-SD-NEXT:    mov v0.s[3], w3
; CHECK-SD-NEXT:    ushl v1.4s, v1.4s, v4.4s
; CHECK-SD-NEXT:    ushl v2.4s, v2.4s, v5.4s
; CHECK-SD-NEXT:    orr v1.16b, v2.16b, v1.16b
; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #3
; CHECK-SD-NEXT:    mov w5, v1.s[1]
; CHECK-SD-NEXT:    mov w6, v1.s[2]
; CHECK-SD-NEXT:    fmov w4, s1
; CHECK-SD-NEXT:    usra v0.4s, v3.4s, #29
; CHECK-SD-NEXT:    mov w1, v0.s[1]
; CHECK-SD-NEXT:    mov w2, v0.s[2]
; CHECK-SD-NEXT:    mov w3, v0.s[3]
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v7i32_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov v0.s[0], w0
; CHECK-GI-NEXT:    mov w8, #29 // =0x1d
; CHECK-GI-NEXT:    mov v2.s[0], w7
; CHECK-GI-NEXT:    mov v1.s[0], w8
; CHECK-GI-NEXT:    mov w9, #3 // =0x3
; CHECK-GI-NEXT:    mov v4.s[0], w4
; CHECK-GI-NEXT:    mov v5.s[0], w9
; CHECK-GI-NEXT:    ldr s3, [sp]
; CHECK-GI-NEXT:    ldr s6, [sp, #24]
; CHECK-GI-NEXT:    ldr s7, [sp, #32]
; CHECK-GI-NEXT:    mov v0.s[1], w1
; CHECK-GI-NEXT:    mov v2.s[1], v3.s[0]
; CHECK-GI-NEXT:    ldr s3, [sp, #8]
; CHECK-GI-NEXT:    mov v1.s[1], w8
; CHECK-GI-NEXT:    mov v6.s[1], v7.s[0]
; CHECK-GI-NEXT:    mov v4.s[1], w5
; CHECK-GI-NEXT:    mov v5.s[1], w9
; CHECK-GI-NEXT:    ldr s7, [sp, #40]
; CHECK-GI-NEXT:    mov v0.s[2], w2
; CHECK-GI-NEXT:    mov v2.s[2], v3.s[0]
; CHECK-GI-NEXT:    ldr s3, [sp, #16]
; CHECK-GI-NEXT:    mov v1.s[2], w8
; CHECK-GI-NEXT:    mov v6.s[2], v7.s[0]
; CHECK-GI-NEXT:    mov v4.s[2], w6
; CHECK-GI-NEXT:    mov v5.s[2], w9
; CHECK-GI-NEXT:    mov v0.s[3], w3
; CHECK-GI-NEXT:    mov v2.s[3], v3.s[0]
; CHECK-GI-NEXT:    neg v1.4s, v1.4s
; CHECK-GI-NEXT:    ushl v3.4s, v4.4s, v5.4s
; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #3
; CHECK-GI-NEXT:    ushl v1.4s, v6.4s, v1.4s
; CHECK-GI-NEXT:    usra v0.4s, v2.4s, #29
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    mov s2, v0.s[1]
; CHECK-GI-NEXT:    mov s3, v0.s[2]
; CHECK-GI-NEXT:    mov s4, v0.s[3]
; CHECK-GI-NEXT:    mov s5, v1.s[1]
; CHECK-GI-NEXT:    mov s6, v1.s[2]
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    fmov w4, s1
; CHECK-GI-NEXT:    fmov w1, s2
; CHECK-GI-NEXT:    fmov w2, s3
; CHECK-GI-NEXT:    fmov w3, s4
; CHECK-GI-NEXT:    fmov w5, s5
; CHECK-GI-NEXT:    fmov w6, s6
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i32> @llvm.fshl(<7 x i32> %a, <7 x i32> %b, <7 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>)
  ret <7 x i32> %d
}

define <7 x i32> @fshr_v7i32_c(<7 x i32> %a, <7 x i32> %b) {
; CHECK-SD-LABEL: fshr_v7i32_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    fmov s0, w0
; CHECK-SD-NEXT:    fmov s2, w4
; CHECK-SD-NEXT:    ldr s1, [sp, #24]
; CHECK-SD-NEXT:    fmov s3, w7
; CHECK-SD-NEXT:    mov x8, sp
; CHECK-SD-NEXT:    add x9, sp, #32
; CHECK-SD-NEXT:    ld1 { v1.s }[1], [x9]
; CHECK-SD-NEXT:    add x9, sp, #40
; CHECK-SD-NEXT:    adrp x10, .LCPI135_1
; CHECK-SD-NEXT:    mov v0.s[1], w1
; CHECK-SD-NEXT:    mov v2.s[1], w5
; CHECK-SD-NEXT:    ldr q5, [x10, :lo12:.LCPI135_1]
; CHECK-SD-NEXT:    ld1 { v3.s }[1], [x8]
; CHECK-SD-NEXT:    add x8, sp, #8
; CHECK-SD-NEXT:    ld1 { v1.s }[2], [x9]
; CHECK-SD-NEXT:    add x9, sp, #16
; CHECK-SD-NEXT:    mov v0.s[2], w2
; CHECK-SD-NEXT:    mov v2.s[2], w6
; CHECK-SD-NEXT:    ld1 { v3.s }[2], [x8]
; CHECK-SD-NEXT:    adrp x8, .LCPI135_0
; CHECK-SD-NEXT:    ldr q4, [x8, :lo12:.LCPI135_0]
; CHECK-SD-NEXT:    ld1 { v3.s }[3], [x9]
; CHECK-SD-NEXT:    mov v0.s[3], w3
; CHECK-SD-NEXT:    ushl v1.4s, v1.4s, v4.4s
; CHECK-SD-NEXT:    ushl v2.4s, v2.4s, v5.4s
; CHECK-SD-NEXT:    orr v1.16b, v2.16b, v1.16b
; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #29
; CHECK-SD-NEXT:    mov w5, v1.s[1]
; CHECK-SD-NEXT:    mov w6, v1.s[2]
; CHECK-SD-NEXT:    fmov w4, s1
; CHECK-SD-NEXT:    usra v0.4s, v3.4s, #3
; CHECK-SD-NEXT:    mov w1, v0.s[1]
; CHECK-SD-NEXT:    mov w2, v0.s[2]
; CHECK-SD-NEXT:    mov w3, v0.s[3]
; CHECK-SD-NEXT:    fmov w0, s0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v7i32_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    mov v0.s[0], w0
; CHECK-GI-NEXT:    mov w8, #3 // =0x3
; CHECK-GI-NEXT:    mov v2.s[0], w7
; CHECK-GI-NEXT:    mov v1.s[0], w8
; CHECK-GI-NEXT:    mov w9, #29 // =0x1d
; CHECK-GI-NEXT:    mov v4.s[0], w4
; CHECK-GI-NEXT:    mov v5.s[0], w9
; CHECK-GI-NEXT:    ldr s3, [sp]
; CHECK-GI-NEXT:    ldr s6, [sp, #24]
; CHECK-GI-NEXT:    ldr s7, [sp, #32]
; CHECK-GI-NEXT:    mov v0.s[1], w1
; CHECK-GI-NEXT:    mov v2.s[1], v3.s[0]
; CHECK-GI-NEXT:    ldr s3, [sp, #8]
; CHECK-GI-NEXT:    mov v1.s[1], w8
; CHECK-GI-NEXT:    mov v6.s[1], v7.s[0]
; CHECK-GI-NEXT:    mov v4.s[1], w5
; CHECK-GI-NEXT:    mov v5.s[1], w9
; CHECK-GI-NEXT:    ldr s7, [sp, #40]
; CHECK-GI-NEXT:    mov v0.s[2], w2
; CHECK-GI-NEXT:    mov v2.s[2], v3.s[0]
; CHECK-GI-NEXT:    ldr s3, [sp, #16]
; CHECK-GI-NEXT:    mov v1.s[2], w8
; CHECK-GI-NEXT:    mov v6.s[2], v7.s[0]
; CHECK-GI-NEXT:    mov v4.s[2], w6
; CHECK-GI-NEXT:    mov v5.s[2], w9
; CHECK-GI-NEXT:    mov v0.s[3], w3
; CHECK-GI-NEXT:    mov v2.s[3], v3.s[0]
; CHECK-GI-NEXT:    neg v1.4s, v1.4s
; CHECK-GI-NEXT:    ushl v3.4s, v4.4s, v5.4s
; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #29
; CHECK-GI-NEXT:    ushl v1.4s, v6.4s, v1.4s
; CHECK-GI-NEXT:    usra v0.4s, v2.4s, #3
; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
; CHECK-GI-NEXT:    mov s2, v0.s[1]
; CHECK-GI-NEXT:    mov s3, v0.s[2]
; CHECK-GI-NEXT:    mov s4, v0.s[3]
; CHECK-GI-NEXT:    mov s5, v1.s[1]
; CHECK-GI-NEXT:    mov s6, v1.s[2]
; CHECK-GI-NEXT:    fmov w0, s0
; CHECK-GI-NEXT:    fmov w4, s1
; CHECK-GI-NEXT:    fmov w1, s2
; CHECK-GI-NEXT:    fmov w2, s3
; CHECK-GI-NEXT:    fmov w3, s4
; CHECK-GI-NEXT:    fmov w5, s5
; CHECK-GI-NEXT:    fmov w6, s6
; CHECK-GI-NEXT:    ret
entry:
  %d = call <7 x i32> @llvm.fshr(<7 x i32> %a, <7 x i32> %b, <7 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>)
  ret <7 x i32> %d
}

define <8 x i32> @fshl_v8i32_c(<8 x i32> %a, <8 x i32> %b) {
; CHECK-SD-LABEL: fshl_v8i32_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.4s, v1.4s, #3
; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #3
; CHECK-SD-NEXT:    usra v1.4s, v3.4s, #29
; CHECK-SD-NEXT:    usra v0.4s, v2.4s, #29
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v8i32_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #3
; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #3
; CHECK-GI-NEXT:    usra v0.4s, v2.4s, #29
; CHECK-GI-NEXT:    usra v1.4s, v3.4s, #29
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i32> @llvm.fshl(<8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>)
  ret <8 x i32> %d
}

define <8 x i32> @fshr_v8i32_c(<8 x i32> %a, <8 x i32> %b) {
; CHECK-SD-LABEL: fshr_v8i32_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.4s, v1.4s, #29
; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #29
; CHECK-SD-NEXT:    usra v1.4s, v3.4s, #3
; CHECK-SD-NEXT:    usra v0.4s, v2.4s, #3
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v8i32_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #29
; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #29
; CHECK-GI-NEXT:    usra v0.4s, v2.4s, #3
; CHECK-GI-NEXT:    usra v1.4s, v3.4s, #3
; CHECK-GI-NEXT:    ret
entry:
  %d = call <8 x i32> @llvm.fshr(<8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>)
  ret <8 x i32> %d
}

define <2 x i64> @fshl_v2i64_c(<2 x i64> %a, <2 x i64> %b) {
; CHECK-LABEL: fshl_v2i64_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    shl v0.2d, v0.2d, #3
; CHECK-NEXT:    usra v0.2d, v1.2d, #61
; CHECK-NEXT:    ret
entry:
  %d = call <2 x i64> @llvm.fshl(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 3, i64 3>)
  ret <2 x i64> %d
}

define <2 x i64> @fshr_v2i64_c(<2 x i64> %a, <2 x i64> %b) {
; CHECK-LABEL: fshr_v2i64_c:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    shl v0.2d, v0.2d, #61
; CHECK-NEXT:    usra v0.2d, v1.2d, #3
; CHECK-NEXT:    ret
entry:
  %d = call <2 x i64> @llvm.fshr(<2 x i64> %a, <2 x i64> %b, <2 x i64> <i64 3, i64 3>)
  ret <2 x i64> %d
}

define <4 x i64> @fshl_v4i64_c(<4 x i64> %a, <4 x i64> %b) {
; CHECK-SD-LABEL: fshl_v4i64_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.2d, v1.2d, #3
; CHECK-SD-NEXT:    shl v0.2d, v0.2d, #3
; CHECK-SD-NEXT:    usra v1.2d, v3.2d, #61
; CHECK-SD-NEXT:    usra v0.2d, v2.2d, #61
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v4i64_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #3
; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #3
; CHECK-GI-NEXT:    usra v0.2d, v2.2d, #61
; CHECK-GI-NEXT:    usra v1.2d, v3.2d, #61
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i64> @llvm.fshl(<4 x i64> %a, <4 x i64> %b, <4 x i64> <i64 3, i64 3, i64 3, i64 3>)
  ret <4 x i64> %d
}

define <4 x i64> @fshr_v4i64_c(<4 x i64> %a, <4 x i64> %b) {
; CHECK-SD-LABEL: fshr_v4i64_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    shl v1.2d, v1.2d, #61
; CHECK-SD-NEXT:    shl v0.2d, v0.2d, #61
; CHECK-SD-NEXT:    usra v1.2d, v3.2d, #3
; CHECK-SD-NEXT:    usra v0.2d, v2.2d, #3
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v4i64_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #61
; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #61
; CHECK-GI-NEXT:    usra v0.2d, v2.2d, #3
; CHECK-GI-NEXT:    usra v1.2d, v3.2d, #3
; CHECK-GI-NEXT:    ret
entry:
  %d = call <4 x i64> @llvm.fshr(<4 x i64> %a, <4 x i64> %b, <4 x i64> <i64 3, i64 3, i64 3, i64 3>)
  ret <4 x i64> %d
}

define <2 x i128> @fshl_v2i128_c(<2 x i128> %a, <2 x i128> %b) {
; CHECK-SD-LABEL: fshl_v2i128_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    extr x8, x0, x5, #61
; CHECK-SD-NEXT:    extr x9, x2, x7, #61
; CHECK-SD-NEXT:    extr x1, x1, x0, #61
; CHECK-SD-NEXT:    extr x3, x3, x2, #61
; CHECK-SD-NEXT:    mov x0, x8
; CHECK-SD-NEXT:    mov x2, x9
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshl_v2i128_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    lsr x8, x5, #61
; CHECK-GI-NEXT:    lsl x9, x1, #3
; CHECK-GI-NEXT:    lsl x10, x3, #3
; CHECK-GI-NEXT:    lsr x11, x7, #61
; CHECK-GI-NEXT:    orr x8, x8, x0, lsl #3
; CHECK-GI-NEXT:    orr x1, x9, x0, lsr #61
; CHECK-GI-NEXT:    orr x3, x10, x2, lsr #61
; CHECK-GI-NEXT:    orr x2, x11, x2, lsl #3
; CHECK-GI-NEXT:    mov x0, x8
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i128> @llvm.fshl(<2 x i128> %a, <2 x i128> %b, <2 x i128> <i128 3, i128 3>)
  ret <2 x i128> %d
}

define <2 x i128> @fshr_v2i128_c(<2 x i128> %a, <2 x i128> %b) {
; CHECK-SD-LABEL: fshr_v2i128_c:
; CHECK-SD:       // %bb.0: // %entry
; CHECK-SD-NEXT:    extr x8, x5, x4, #3
; CHECK-SD-NEXT:    extr x9, x7, x6, #3
; CHECK-SD-NEXT:    extr x1, x0, x5, #3
; CHECK-SD-NEXT:    extr x3, x2, x7, #3
; CHECK-SD-NEXT:    mov x0, x8
; CHECK-SD-NEXT:    mov x2, x9
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: fshr_v2i128_c:
; CHECK-GI:       // %bb.0: // %entry
; CHECK-GI-NEXT:    lsl x8, x5, #61
; CHECK-GI-NEXT:    lsl x9, x7, #61
; CHECK-GI-NEXT:    lsr x10, x5, #3
; CHECK-GI-NEXT:    lsr x11, x7, #3
; CHECK-GI-NEXT:    orr x8, x8, x4, lsr #3
; CHECK-GI-NEXT:    orr x9, x9, x6, lsr #3
; CHECK-GI-NEXT:    orr x1, x10, x0, lsl #61
; CHECK-GI-NEXT:    orr x3, x11, x2, lsl #61
; CHECK-GI-NEXT:    mov x0, x8
; CHECK-GI-NEXT:    mov x2, x9
; CHECK-GI-NEXT:    ret
entry:
  %d = call <2 x i128> @llvm.fshr(<2 x i128> %a, <2 x i128> %b, <2 x i128> <i128 3, i128 3>)
  ret <2 x i128> %d
}