llvm/test/CodeGen/AArch64/sme-peephole-opts.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-new-sme-abi -aarch64-streaming-hazard-size=0 -mattr=+sve,+sme2 < %s | FileCheck %s

declare void @callee()
declare void @callee_sm() "aarch64_pstate_sm_enabled"
declare void @callee_farg(float)
declare float @callee_farg_fret(float)

; normal caller -> streaming callees
define void @test0(ptr %callee) nounwind {
; CHECK-LABEL: test0:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Spill
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    bl callee_sm
; CHECK-NEXT:    bl callee_sm
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  call void @callee_sm()
  call void @callee_sm()
  ret void
}

; streaming caller -> normal callees
define void @test1() nounwind "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: test1:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Spill
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  call void @callee()
  call void @callee()
  ret void
}

; streaming-compatible caller -> normal callees
define void @test2() nounwind "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: test2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT:    mrs x19, SVCR
; CHECK-NEXT:    tbz w19, #0, .LBB2_2
; CHECK-NEXT:  // %bb.1:
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:  .LBB2_2:
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    tbz w19, #0, .LBB2_4
; CHECK-NEXT:  // %bb.3:
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:  .LBB2_4:
; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  call void @callee()
  call void @callee()
  ret void
}

; streaming-compatible caller -> mixed callees
define void @test3() nounwind "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: test3:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT:    mrs x19, SVCR
; CHECK-NEXT:    tbnz w19, #0, .LBB3_2
; CHECK-NEXT:  // %bb.1:
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:  .LBB3_2:
; CHECK-NEXT:    bl callee_sm
; CHECK-NEXT:    tbnz w19, #0, .LBB3_4
; CHECK-NEXT:  // %bb.3:
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:  .LBB3_4:
; CHECK-NEXT:    tbz w19, #0, .LBB3_6
; CHECK-NEXT:  // %bb.5:
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:  .LBB3_6:
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    tbz w19, #0, .LBB3_8
; CHECK-NEXT:  // %bb.7:
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:  .LBB3_8:
; CHECK-NEXT:    tbnz w19, #0, .LBB3_10
; CHECK-NEXT:  // %bb.9:
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:  .LBB3_10:
; CHECK-NEXT:    bl callee_sm
; CHECK-NEXT:    tbnz w19, #0, .LBB3_12
; CHECK-NEXT:  // %bb.11:
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:  .LBB3_12:
; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  call void @callee_sm()
  call void @callee()
  call void @callee_sm()
  ret void
}

; streaming caller -> normal callees (pass 0.0f)
define void @test4() nounwind "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: test4:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Spill
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    fmov s0, wzr
; CHECK-NEXT:    bl callee_farg
; CHECK-NEXT:    fmov s0, wzr
; CHECK-NEXT:    bl callee_farg
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  call void @callee_farg(float zeroinitializer)
  call void @callee_farg(float zeroinitializer)
  ret void
}

; streaming caller -> normal callees (pass fp arg)
define void @test5(float %f) nounwind "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: test5:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #96
; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Spill
; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Spill
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Reload
; CHECK-NEXT:    bl callee_farg
; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Reload
; CHECK-NEXT:    bl callee_farg
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    add sp, sp, #96
; CHECK-NEXT:    ret
  call void @callee_farg(float %f)
  call void @callee_farg(float %f)
  ret void
}

define float @test6(float %f) nounwind "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: test6:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #96
; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Spill
; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Spill
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Reload
; CHECK-NEXT:    bl callee_farg_fret
; CHECK-NEXT:    bl callee_farg_fret
; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Spill
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    add sp, sp, #96
; CHECK-NEXT:    ret
  %res0 = call float @callee_farg_fret(float %f)
  %res1 = call float @callee_farg_fret(float %res0)
  ret float %res1
}

; save/restore zt0 to stack is not yet optimised away by the pass,
; because of the ldr/str of zt0, which will need some further analysis
; to make sure if the redundant str can be removed.
define void @test7() nounwind "aarch64_inout_zt0" {
; CHECK-LABEL: test7:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #80
; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT:    mov x19, sp
; CHECK-NEXT:    str zt0, [x19]
; CHECK-NEXT:    smstop za
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    smstart za
; CHECK-NEXT:    ldr zt0, [x19]
; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
; CHECK-NEXT:    add sp, sp, #80
; CHECK-NEXT:    ret
  call void @callee()
  call void @callee()
  ret void
}

; test that 'smstop za' is not cancelled out with 'smstart sm'.
define void @test8() nounwind "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: test8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Spill
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    smstop za
; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  call void @callee()
  call void @llvm.aarch64.sme.za.disable()
  ret void
}

; test that the 'smstart' and 'smstop' are entirely removed,
; along with any code to read 'vg' for the CFI.
define void @test9() "aarch64_pstate_sm_body" {
; CHECK-LABEL: test9:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT:    .cfi_def_cfa_offset 16
; CHECK-NEXT:    .cfi_offset w30, -8
; CHECK-NEXT:    .cfi_offset w29, -16
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  call void @callee()
  ret void
}

; test that if the 'smstart' and 'smtop' are entirely removed in a locally
; streaming function, we use the FP, not an expression to describe the CFA.
define aarch64_sve_vector_pcs void @test9_1() "aarch64_pstate_sm_body" {
; CHECK-LABEL: test9_1:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
; CHECK-NEXT:    str x28, [sp, #16] // 8-byte Spill
; CHECK-NEXT:    mov x29, sp
; CHECK-NEXT:    addsvl sp, sp, #-18
; CHECK-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Spill
; CHECK-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Spill
; CHECK-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Spill
; CHECK-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Spill
; CHECK-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Spill
; CHECK-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Spill
; CHECK-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Spill
; CHECK-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Spill
; CHECK-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Spill
; CHECK-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Spill
; CHECK-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Spill
; CHECK-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Spill
; CHECK-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
; CHECK-NEXT:    .cfi_def_cfa w29, 32
; CHECK-NEXT:    .cfi_offset w28, -16
; CHECK-NEXT:    .cfi_offset w30, -24
; CHECK-NEXT:    .cfi_offset w29, -32
; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d8 @ cfa - 8 * VG - 32
; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d9 @ cfa - 16 * VG - 32
; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d10 @ cfa - 24 * VG - 32
; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d11 @ cfa - 32 * VG - 32
; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d12 @ cfa - 40 * VG - 32
; CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d13 @ cfa - 48 * VG - 32
; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d14 @ cfa - 56 * VG - 32
; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d15 @ cfa - 64 * VG - 32
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
; CHECK-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Reload
; CHECK-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Reload
; CHECK-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Reload
; CHECK-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Reload
; CHECK-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Reload
; CHECK-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Reload
; CHECK-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Reload
; CHECK-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Reload
; CHECK-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Reload
; CHECK-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Reload
; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Reload
; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Reload
; CHECK-NEXT:    addsvl sp, sp, #18
; CHECK-NEXT:    ldr x28, [sp, #16] // 8-byte Reload
; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  call void @callee()
  ret void
}

; similar to above, but in this case only the first
; 'smstart, smstop' pair can be removed and the code required
; for the CFI is still needed.
define void @test10() "aarch64_pstate_sm_body" {
; CHECK-LABEL: test10:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; CHECK-NEXT:    .cfi_def_cfa_offset 96
; CHECK-NEXT:    cntd x9
; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Spill
; CHECK-NEXT:    .cfi_offset vg, -16
; CHECK-NEXT:    .cfi_offset w30, -24
; CHECK-NEXT:    .cfi_offset w29, -32
; CHECK-NEXT:    .cfi_offset b8, -40
; CHECK-NEXT:    .cfi_offset b9, -48
; CHECK-NEXT:    .cfi_offset b10, -56
; CHECK-NEXT:    .cfi_offset b11, -64
; CHECK-NEXT:    .cfi_offset b12, -72
; CHECK-NEXT:    .cfi_offset b13, -80
; CHECK-NEXT:    .cfi_offset b14, -88
; CHECK-NEXT:    .cfi_offset b15, -96
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    bl callee_sm
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; CHECK-NEXT:    .cfi_def_cfa_offset 0
; CHECK-NEXT:    .cfi_restore vg
; CHECK-NEXT:    .cfi_restore w30
; CHECK-NEXT:    .cfi_restore w29
; CHECK-NEXT:    .cfi_restore b8
; CHECK-NEXT:    .cfi_restore b9
; CHECK-NEXT:    .cfi_restore b10
; CHECK-NEXT:    .cfi_restore b11
; CHECK-NEXT:    .cfi_restore b12
; CHECK-NEXT:    .cfi_restore b13
; CHECK-NEXT:    .cfi_restore b14
; CHECK-NEXT:    .cfi_restore b15
; CHECK-NEXT:    ret
  call void @callee()
  call void @callee_sm()
  call void @callee()
  ret void
}

; test that an operation like a store is executed in the right
; streaming mode and blocks the optimization.
define void @test11(ptr %p) nounwind "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: test11:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT:    mov x19, x0
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    mov z0.b, #0 // =0x0
; CHECK-NEXT:    str z0, [x19]
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  call void @callee()
  store <vscale x 16 x i8> zeroinitializer, ptr %p
  call void @callee()
  ret void
}

; test that 'smstart sm' and 'smstop za' don't get folded away together.
; we can further optimize this test by considering streaming mode
; separately from ZA.
define void @test12() "aarch64_pstate_sm_body" {
; CHECK-LABEL: test12:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; CHECK-NEXT:    .cfi_def_cfa_offset 96
; CHECK-NEXT:    cntd x9
; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Spill
; CHECK-NEXT:    .cfi_offset vg, -16
; CHECK-NEXT:    .cfi_offset w30, -24
; CHECK-NEXT:    .cfi_offset w29, -32
; CHECK-NEXT:    .cfi_offset b8, -40
; CHECK-NEXT:    .cfi_offset b9, -48
; CHECK-NEXT:    .cfi_offset b10, -56
; CHECK-NEXT:    .cfi_offset b11, -64
; CHECK-NEXT:    .cfi_offset b12, -72
; CHECK-NEXT:    .cfi_offset b13, -80
; CHECK-NEXT:    .cfi_offset b14, -88
; CHECK-NEXT:    .cfi_offset b15, -96
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    smstop za
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    smstart za
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; CHECK-NEXT:    .cfi_def_cfa_offset 0
; CHECK-NEXT:    .cfi_restore vg
; CHECK-NEXT:    .cfi_restore w30
; CHECK-NEXT:    .cfi_restore w29
; CHECK-NEXT:    .cfi_restore b8
; CHECK-NEXT:    .cfi_restore b9
; CHECK-NEXT:    .cfi_restore b10
; CHECK-NEXT:    .cfi_restore b11
; CHECK-NEXT:    .cfi_restore b12
; CHECK-NEXT:    .cfi_restore b13
; CHECK-NEXT:    .cfi_restore b14
; CHECK-NEXT:    .cfi_restore b15
; CHECK-NEXT:    ret
  call void @llvm.aarch64.sme.za.disable()
  call void @callee()
  call void @llvm.aarch64.sme.za.enable()
  ret void
}

; We conservatively don't remove the smstart/smstop pair yet when there are COPY
; instructions that copy SVE registers, because we can't yet conclusively prove
; that this is safe (although for this example, it would be).
define void @test13(ptr %ptr) nounwind "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: test13:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Spill
; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
; CHECK-NEXT:    addvl sp, sp, #-1
; CHECK-NEXT:    mov z0.s, #0 // =0x0
; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    rdvl x8, #1
; CHECK-NEXT:    addsvl x8, x8, #-1
; CHECK-NEXT:    cbnz x8, .LBB14_2
; CHECK-NEXT:  // %bb.1:
; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
; CHECK-NEXT:    mov x19, x0
; CHECK-NEXT:    bl callee_farg_fret
; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    rdvl x8, #1
; CHECK-NEXT:    addsvl x8, x8, #-1
; CHECK-NEXT:    cbz x8, .LBB14_3
; CHECK-NEXT:  .LBB14_2:
; CHECK-NEXT:    brk #0x1
; CHECK-NEXT:  .LBB14_3:
; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
; CHECK-NEXT:    bl callee_farg_fret
; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
; CHECK-NEXT:    str z0, [x19]
; CHECK-NEXT:    addvl sp, sp, #1
; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Reload
; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  %res0 = call <vscale x 4 x float> @callee_farg_fret(<vscale x 4 x float> zeroinitializer)
  %res1 = call <vscale x 4 x float> @callee_farg_fret(<vscale x 4 x float> %res0)
  store <vscale x 4 x float> %res1, ptr %ptr
  ret void
}

; normal caller -> streaming callees (with ZA state)
define void @test14(ptr %callee) nounwind "aarch64_inout_za" {
; CHECK-LABEL: test14:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT:    add x29, sp, #64
; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Spill
; CHECK-NEXT:    sub sp, sp, #16
; CHECK-NEXT:    rdsvl x8, #1
; CHECK-NEXT:    mov x9, sp
; CHECK-NEXT:    msub x9, x8, x8, x9
; CHECK-NEXT:    mov sp, x9
; CHECK-NEXT:    sub x10, x29, #80
; CHECK-NEXT:    stp x9, x8, [x29, #-80]
; CHECK-NEXT:    msr TPIDR2_EL0, x10
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    bl callee_sm
; CHECK-NEXT:    bl callee_sm
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    smstart za
; CHECK-NEXT:    mrs x8, TPIDR2_EL0
; CHECK-NEXT:    sub x0, x29, #80
; CHECK-NEXT:    cbnz x8, .LBB15_2
; CHECK-NEXT:  // %bb.1:
; CHECK-NEXT:    bl __arm_tpidr2_restore
; CHECK-NEXT:  .LBB15_2:
; CHECK-NEXT:    msr TPIDR2_EL0, xzr
; CHECK-NEXT:    sub sp, x29, #64
; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Reload
; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  call void @callee_sm()
  call void @callee_sm()
  ret void
}

; normal caller -> streaming callees (with ZA agnostic state)
define void @test15(ptr %callee) nounwind "aarch64_za_state_agnostic" {
; CHECK-LABEL: test15:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT:    add x29, sp, #64
; CHECK-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
; CHECK-NEXT:    bl __arm_sme_state_size
; CHECK-NEXT:    sub sp, sp, x0
; CHECK-NEXT:    mov x20, sp
; CHECK-NEXT:    mov x0, x20
; CHECK-NEXT:    bl __arm_sme_save
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    bl callee_sm
; CHECK-NEXT:    bl callee_sm
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    mov x0, x20
; CHECK-NEXT:    bl __arm_sme_restore
; CHECK-NEXT:    sub sp, x29, #64
; CHECK-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  call void @callee_sm()
  call void @callee_sm()
  ret void
}

; locally streaming caller -> normal callees (with ZA state)
define void @test16(ptr %callee) nounwind "aarch64_pstate_sm_body" "aarch64_new_za" {
; CHECK-LABEL: test16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
; CHECK-NEXT:    add x29, sp, #64
; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Spill
; CHECK-NEXT:    sub sp, sp, #16
; CHECK-NEXT:    rdsvl x8, #1
; CHECK-NEXT:    mov x9, sp
; CHECK-NEXT:    msub x9, x8, x8, x9
; CHECK-NEXT:    mov sp, x9
; CHECK-NEXT:    stp x9, x8, [x29, #-80]
; CHECK-NEXT:    mrs x8, TPIDR2_EL0
; CHECK-NEXT:    cbz x8, .LBB17_2
; CHECK-NEXT:  // %bb.1:
; CHECK-NEXT:    bl __arm_tpidr2_save
; CHECK-NEXT:    msr TPIDR2_EL0, xzr
; CHECK-NEXT:    zero {za}
; CHECK-NEXT:  .LBB17_2:
; CHECK-NEXT:    smstart za
; CHECK-NEXT:    smstart sm
; CHECK-NEXT:    sub x8, x29, #80
; CHECK-NEXT:    msr TPIDR2_EL0, x8
; CHECK-NEXT:    smstop sm
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    bl callee
; CHECK-NEXT:    msr TPIDR2_EL0, xzr
; CHECK-NEXT:    smstop za
; CHECK-NEXT:    sub sp, x29, #64
; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Reload
; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
; CHECK-NEXT:    ret
  call void @callee()
  call void @callee()
  ret void
}