llvm/test/CodeGen/AMDGPU/sdwa-ops.mir

# RUN: llc -mtriple=amdgpu9.00 -run-pass=si-peephole-sdwa -o - %s | FileCheck -check-prefix=GFX9 %s
# RUN: llc -mtriple=amdgpu8.03 -run-pass=si-peephole-sdwa -o - %s | FileCheck -check-prefix=GFX9 %s
# RUN: llc -mtriple=amdgpu9.00 -passes=si-peephole-sdwa -o - %s | FileCheck -check-prefix=GFX9 %s
# RUN: llc -mtriple=amdgpu8.03 -passes=si-peephole-sdwa -o - %s | FileCheck -check-prefix=GFX9 %s

# test for 3 consecutive _sdwa's
# GFX9-LABEL: name:            test1_add_co_sdwa
# GFX9: = nsw V_ADD_CO_U32_sdwa
# GFX9-NEXT: = nuw V_ADDC_U32_e64
# GFX9: V_ADD_CO_U32_sdwa
# GFX9-NEXT: V_ADDC_U32_e64
# GFX9: V_ADD_CO_U32_sdwa
# GFX9-NEXT: V_ADDC_U32_e64
---
name:            test1_add_co_sdwa
tracksRegLiveness: true
registers:
  - { id: 0, class: vgpr_32, preferred-register: '' }
liveins:
  - { reg: '$vgpr0', virtual-reg: '%0' }
  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
body:             |
  bb.0:
    liveins: $vgpr0, $sgpr0_sgpr1

    %1:sgpr_64 = COPY $sgpr0_sgpr1
    %0:vgpr_32 = COPY $vgpr0
    %2:sreg_32_xm0 = S_MOV_B32 255
    %3:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %4:vreg_64 = COPY $sgpr0_sgpr1
    %5:vgpr_32, %6:sreg_64_xexec = nsw V_ADD_CO_U32_e64 %4.sub0, %3, 0, implicit $exec
    %7:vgpr_32, dead %8:sreg_64_xexec = nuw V_ADDC_U32_e64 %4.sub1, %0, %6, 0, implicit $exec
    %9:vreg_64 = REG_SEQUENCE %5, %subreg.sub0, %7, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %4.sub0, %9, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))

    %10:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %11:vgpr_32, %12:sreg_64_xexec = V_ADD_CO_U32_e64 %4.sub0, %10, 0, implicit $exec
    %13:vgpr_32, dead %14:sreg_64_xexec = V_ADDC_U32_e64 %4.sub1, %0, %12, 0, implicit $exec
    %15:vreg_64 = REG_SEQUENCE %11, %subreg.sub0, %13, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %4.sub0, %15, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))

    %16:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %17:vgpr_32, %18:sreg_64_xexec = V_ADD_CO_U32_e64 %4.sub0, %16, 0, implicit $exec
    %19:vgpr_32, dead %20:sreg_64_xexec = V_ADDC_U32_e64 %4.sub1, %0, %18, 0, implicit $exec
    %21:vreg_64 = REG_SEQUENCE %17, %subreg.sub0, %19, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %4.sub0, %21, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))

...

# test for VCC interference on sdwa, should generate 1 xform only
# GFX9-LABEL: name:            test2_add_co_sdwa
# GFX9: V_ADD_CO_U32_sdwa
# GFX9: V_ADDC_U32_e64
# GFX9-NOT: V_ADD_CO_U32_sdwa
# GFX9-NOT: V_ADDC_U32_e32
---
name:            test2_add_co_sdwa
tracksRegLiveness: true
registers:
  - { id: 0, class: vgpr_32, preferred-register: '' }
liveins:
  - { reg: '$vgpr0', virtual-reg: '%0' }
  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
body:             |
  bb.0:
    liveins: $vgpr0, $sgpr0_sgpr1

    %1:sgpr_64 = COPY $sgpr0_sgpr1
    %0:vgpr_32 = COPY $vgpr0
    %2:sreg_32_xm0 = S_MOV_B32 255
    %3:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %4:vreg_64 = COPY $sgpr0_sgpr1
    %5:vgpr_32, %6:sreg_64_xexec = V_ADD_CO_U32_e64 %4.sub0, %3, 0, implicit $exec

    %7:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %8:vgpr_32, %9:sreg_64_xexec = V_ADD_CO_U32_e64 %4.sub0, %7, 0, implicit $exec
    %10:vgpr_32, dead %11:sreg_64_xexec = V_ADDC_U32_e64 %4.sub1, %0, %9, 0, implicit $exec
    %12:vreg_64 = REG_SEQUENCE %8, %subreg.sub0, %10, %subreg.sub1

    %13:vgpr_32, dead %14:sreg_64_xexec = V_ADDC_U32_e64 %4.sub1, %0, %6, 0, implicit $exec
    %15:vreg_64 = REG_SEQUENCE %5, %subreg.sub0, %13, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %4.sub0, %15, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))

    %7:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %8:vgpr_32, %9:sreg_64_xexec = V_ADD_CO_U32_e64 %4.sub0, %7, 0, implicit $exec
    %10:vgpr_32, dead %11:sreg_64_xexec = V_ADDC_U32_e64 %4.sub1, %0, %9, 0, implicit $exec
    %12:vreg_64 = REG_SEQUENCE %8, %subreg.sub0, %10, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %4.sub0, %12, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))

...

# test for CarryOut used, should reject
# GFX9-LABEL: name:            test3_add_co_sdwa
# GFX9: V_ADD_CO_U32_e64
# GFX9: V_ADDC_U32_e64
# GFX9-NOT: V_ADD_CO_U32_sdwa
# GFX9-NOT: V_ADDC_U32_e32
---
name:            test3_add_co_sdwa
tracksRegLiveness: true
registers:
  - { id: 0, class: vgpr_32, preferred-register: '' }
liveins:
  - { reg: '$vgpr0', virtual-reg: '%0' }
  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
body:             |
  bb.0:
    liveins: $vgpr0, $sgpr0_sgpr1

    %1:sgpr_64 = COPY $sgpr0_sgpr1
    %0:vgpr_32 = COPY $vgpr0
    %2:sreg_32_xm0 = S_MOV_B32 255
    %3:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %4:vreg_64 = COPY $sgpr0_sgpr1
    %5:vgpr_32, %6:sreg_64_xexec = V_ADD_CO_U32_e64 %4.sub0, %3, 0, implicit $exec
    %7:vgpr_32, %8:sreg_64_xexec = V_ADDC_U32_e64 %4.sub1, %0, %6, 0, implicit $exec
    %9:vreg_64 = REG_SEQUENCE %5, %subreg.sub0, %8, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %4.sub0, %9, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))

...

# test for CarryIn used more than once, should reject
# GFX9-LABEL: name:            test4_add_co_sdwa
# GFX9: V_ADD_CO_U32_e64
# GFX9: V_ADDC_U32_e64
# GFX9-NOT: V_ADD_CO_U32_sdwa
# GFX9-NOT: V_ADDC_U32_e32
---
name:            test4_add_co_sdwa
tracksRegLiveness: true
registers:
  - { id: 0, class: vgpr_32, preferred-register: '' }
liveins:
  - { reg: '$vgpr0', virtual-reg: '%0' }
  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
body:             |
  bb.0:
    liveins: $vgpr0, $sgpr0_sgpr1

    %1:sgpr_64 = COPY $sgpr0_sgpr1
    %0:vgpr_32 = COPY $vgpr0
    %2:sreg_32_xm0 = S_MOV_B32 255
    %3:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %4:vreg_64 = COPY $sgpr0_sgpr1
    %5:vgpr_32, %6:sreg_64_xexec = V_ADD_CO_U32_e64 %4.sub0, %3, 0, implicit $exec
    %7:vgpr_32, %8:sreg_64_xexec = V_ADDC_U32_e64 %4.sub1, %0, %6, 0, implicit $exec
    %9:vreg_64 = REG_SEQUENCE %5, %subreg.sub0, %6, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %4.sub0, %9, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))


...

# test for simple example, should generate sdwa
# GFX9-LABEL: name:            test5_add_co_sdwa
# GFX9: V_ADD_CO_U32_sdwa
# GFX9: V_ADDC_U32_e64
---
name:            test5_add_co_sdwa
tracksRegLiveness: true
registers:
  - { id: 0, class: vgpr_32, preferred-register: '' }
liveins:
  - { reg: '$vgpr0', virtual-reg: '%0' }
  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
body:             |
  bb.0:
    liveins: $vgpr0, $sgpr0_sgpr1

    %1:sgpr_64 = COPY $sgpr0_sgpr1
    %0:vgpr_32 = COPY $vgpr0
    %2:sreg_32_xm0 = S_MOV_B32 255
    %3:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %4:vreg_64 = COPY $sgpr0_sgpr1
    %5:vgpr_32, %6:sreg_64_xexec = V_ADD_CO_U32_e64 %4.sub0, %3, 0, implicit $exec
    %7:vgpr_32, %8:sreg_64_xexec = V_ADDC_U32_e64 %4.sub1, %0, %6, 0, implicit $exec
    %9:vreg_64 = REG_SEQUENCE %5, %subreg.sub0, %7, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %4.sub0, %9, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))


...

# test for V_ADD_CO_U32_e64 only, should reject
# GFX9-LABEL: name:            test6_add_co_sdwa
# GFX9: V_ADD_CO_U32_e64
# GFX9-NOT: V_ADD_CO_U32_sdwa
# GFX9-NOT: V_ADDC_U32_e32
---
name:            test6_add_co_sdwa
tracksRegLiveness: true
registers:
  - { id: 0, class: vgpr_32, preferred-register: '' }
liveins:
  - { reg: '$vgpr0', virtual-reg: '%0' }
  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
body:             |
  bb.0:
    liveins: $vgpr0, $sgpr0_sgpr1

    %1:sgpr_64 = COPY $sgpr0_sgpr1
    %0:vgpr_32 = COPY $vgpr0
    %2:sreg_32_xm0 = S_MOV_B32 255
    %3:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %4:vreg_64 = COPY $sgpr0_sgpr1
    %5:vgpr_32, %6:sreg_64_xexec = V_ADD_CO_U32_e64 %4.sub0, %3, 0, implicit $exec
    %7:vreg_64 = REG_SEQUENCE %5, %subreg.sub0, %3, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %4.sub0, %7, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))


...

# test for V_ADDC_U32_e64 only, should reject
# GFX9-LABEL: name:            test7_add_co_sdwa
# GFX9: V_ADDC_U32_e64
# GFX9-NOT: V_ADD_CO_U32_sdwa
# GFX9-NOT: V_ADDC_U32_e32
---
name:            test7_add_co_sdwa
tracksRegLiveness: true
registers:
  - { id: 0, class: vgpr_32, preferred-register: '' }
liveins:
  - { reg: '$vgpr0', virtual-reg: '%0' }
  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
body:             |
  bb.0:
    liveins: $vgpr0, $sgpr0_sgpr1

    %1:sgpr_64 = COPY $sgpr0_sgpr1
    %0:vgpr_32 = COPY $vgpr0
    %2:sreg_32_xm0 = S_MOV_B32 255
    %3:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %4:sreg_64_xexec = COPY $sgpr0_sgpr1

    %5:vreg_64 = COPY $sgpr0_sgpr1
    %6:vgpr_32, %7:sreg_64_xexec = V_ADDC_U32_e64 %5.sub1, %0, %4, 0, implicit $exec
    %8:vreg_64 = REG_SEQUENCE %3, %subreg.sub0, %3, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %5.sub0, %8, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))


...

# test for $vcc defined between two adds, should not generate
# GFX9-LABEL: name:            test8_add_co_sdwa
# GFX9-NOT: V_ADD_CO_U32_sdwa
# GFX9: V_ADDC_U32_e64
---
name:            test8_add_co_sdwa
tracksRegLiveness: true
registers:
  - { id: 0, class: vgpr_32, preferred-register: '' }
liveins:
  - { reg: '$vgpr0', virtual-reg: '%0' }
  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
body:             |
  bb.0:
    liveins: $vgpr0, $sgpr0_sgpr1

    %1:sgpr_64 = COPY $sgpr0_sgpr1
    %0:vgpr_32 = COPY $vgpr0
    %2:sreg_32_xm0 = S_MOV_B32 255
    %3:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %4:vreg_64 = COPY $sgpr0_sgpr1
    %5:vgpr_32, %6:sreg_64_xexec = V_ADD_CO_U32_e64 %4.sub0, %3, 0, implicit $exec
    $vcc = COPY %4
    %7:vgpr_32, %8:sreg_64_xexec = V_ADDC_U32_e64 %4.sub1, %0, %6, 0, implicit $exec
    %9:vreg_64 = COPY $vcc
    %10:vreg_64 = REG_SEQUENCE %5, %subreg.sub0, %7, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %9.sub0, %10, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))


...

# test for non dead $vcc, should not generate
# GFX9-LABEL: name:            test9_add_co_sdwa
# GFX9-NOT: V_ADD_CO_U32_sdwa
# GFX9: V_ADDC_U32_e64
---
name:            test9_add_co_sdwa
tracksRegLiveness: true
registers:
  - { id: 0, class: vgpr_32, preferred-register: '' }
liveins:
  - { reg: '$vgpr0', virtual-reg: '%0' }
  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
body:             |
  bb.0:
    liveins: $vgpr0, $sgpr0_sgpr1

    %1:sgpr_64 = COPY $sgpr0_sgpr1
    %0:vgpr_32 = COPY $vgpr0
    %2:sreg_32_xm0 = S_MOV_B32 255
    %3:vreg_64 = COPY $sgpr0_sgpr1
    $vcc = COPY %3
    %4:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %5:vgpr_32, %6:sreg_64_xexec = V_ADD_CO_U32_e64 %3.sub0, %4, 0, implicit $exec
    %7:vgpr_32, %8:sreg_64_xexec = V_ADDC_U32_e64 %3.sub1, %0, %6, 0, implicit $exec
    %9:vreg_64 = COPY $vcc
    %10:vreg_64 = REG_SEQUENCE %5, %subreg.sub0, %7, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %9.sub0, %10, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))


...

# test for def $vcc_lo, should not generate
# GFX9-LABEL: name:            test10_add_co_sdwa
# GFX9-NOT: V_ADD_CO_U32_sdwa
# GFX9: V_ADDC_U32_e64
---
name:            test10_add_co_sdwa
tracksRegLiveness: true
registers:
  - { id: 0, class: vgpr_32, preferred-register: '' }
liveins:
  - { reg: '$vgpr0', virtual-reg: '%0' }
  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
body:             |
  bb.0:
    liveins: $vgpr0, $sgpr0_sgpr1

    %1:sgpr_64 = COPY $sgpr0_sgpr1
    %0:vgpr_32 = COPY $vgpr0
    %2:sreg_32_xm0 = S_MOV_B32 255
    %3:vreg_64 = COPY $sgpr0_sgpr1
    $vcc_lo = COPY %3.sub0
    %4:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %5:vgpr_32, %6:sreg_64_xexec = V_ADD_CO_U32_e64 %3.sub0, %4, 0, implicit $exec
    %7:vgpr_32 = COPY $vcc_lo
    %8:vreg_64 = REG_SEQUENCE %7, %subreg.sub0, %4, %subreg.sub1
    %9:vgpr_32, %10:sreg_64_xexec = V_ADDC_U32_e64 %3.sub1, %0, %6, 0, implicit $exec
    %11:vreg_64 = REG_SEQUENCE %5, %subreg.sub0, %9, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %8.sub0, %11, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))

...

# test for read $vcc_hi, should not generate
# GFX9-LABEL: name:            test11_add_co_sdwa
# GFX9-NOT: V_ADD_CO_U32_sdwa
# GFX9: V_ADDC_U32_e64
---
name:            test11_add_co_sdwa
tracksRegLiveness: true
registers:
  - { id: 0, class: vgpr_32, preferred-register: '' }
liveins:
  - { reg: '$vgpr0', virtual-reg: '%0' }
  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
body:             |
  bb.0:
    liveins: $vgpr0, $sgpr0_sgpr1

    %1:sgpr_64 = COPY $sgpr0_sgpr1
    %0:vgpr_32 = COPY $vgpr0
    %2:sreg_32_xm0 = S_MOV_B32 255
    %3:vreg_64 = COPY $sgpr0_sgpr1
    $vcc_hi = COPY %3.sub0
    %4:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %5:vgpr_32, %6:sreg_64_xexec = V_ADD_CO_U32_e64 %3.sub0, %4, 0, implicit $exec
    %7:vgpr_32 = COPY $vcc_hi
    %8:vreg_64 = REG_SEQUENCE %7, %subreg.sub0, %4, %subreg.sub1
    %9:vgpr_32, %10:sreg_64_xexec = V_ADDC_U32_e64 %3.sub1, %0, %6, 0, implicit $exec
    %11:vreg_64 = REG_SEQUENCE %5, %subreg.sub0, %9, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %8.sub0, %11, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))

...

# test for $vcc defined and used between adds, should not generate
# GFX9-LABEL: name:            test12_add_co_sdwa
# GFX9-NOT: V_ADD_CO_U32_sdwa
# GFX9: V_ADDC_U32_e64
---
name:            test12_add_co_sdwa
tracksRegLiveness: true
registers:
  - { id: 0, class: vgpr_32, preferred-register: '' }
liveins:
  - { reg: '$vgpr0', virtual-reg: '%0' }
  - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
body:             |
  bb.0:
    liveins: $vgpr0, $sgpr0_sgpr1

    %1:sgpr_64 = COPY $sgpr0_sgpr1
    %0:vgpr_32 = COPY $vgpr0
    %2:sreg_32_xm0 = S_MOV_B32 255
    %3:vreg_64 = COPY $sgpr0_sgpr1
    %4:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %5:vgpr_32, %6:sreg_64_xexec = V_ADD_CO_U32_e64 %3.sub0, %4, 0, implicit $exec
    $vcc = COPY %3
    %7:vreg_64 = COPY killed $vcc
    %8:vgpr_32, %9:sreg_64_xexec = V_ADDC_U32_e64 %3.sub1, %0, %6, 0, implicit $exec
    %10:vreg_64 = REG_SEQUENCE %5, %subreg.sub0, %8, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %7.sub0, %10, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))

...

# MachineSink can move the ADDC into a successor when its carry-out is dead,
# leaving the ADD in the predecessor. Walking (MI, MISucc] then leaves the
# ADD's MBB and used to dereference the block sentinel. The pair must stay e64.
# GFX9-LABEL: name:            test13_add_co_addc_different_mbb
# GFX9: V_ADD_CO_U32_e64
# GFX9-NEXT: S_BRANCH
# GFX9: V_ADDC_U32_e64
# GFX9-NEXT: REG_SEQUENCE
---
name:            test13_add_co_addc_different_mbb
tracksRegLiveness: true
body:             |
  bb.0:
    successors: %bb.1
    liveins: $vgpr0, $sgpr0_sgpr1

    %1:sgpr_64 = COPY $sgpr0_sgpr1
    %0:vgpr_32 = COPY $vgpr0
    %2:sreg_32_xm0 = S_MOV_B32 255
    %3:vgpr_32 = V_AND_B32_e32 %2, %0, implicit $exec
    %4:vreg_64 = COPY $sgpr0_sgpr1
    %5:vgpr_32, %6:sreg_64_xexec = V_ADD_CO_U32_e64 %4.sub0, %3, 0, implicit $exec
    S_BRANCH %bb.1

  bb.1:
    liveins: $vgpr0, $sgpr0_sgpr1

    %7:vgpr_32, dead %8:sreg_64_xexec = V_ADDC_U32_e64 %4.sub1, %0, %6, 0, implicit $exec
    %9:vreg_64 = REG_SEQUENCE %5, %subreg.sub0, %7, %subreg.sub1
    GLOBAL_STORE_DWORDX2_SADDR %4.sub0, %9, %1, 0, 0, implicit $exec, implicit $exec :: (store (s64))
    S_ENDPGM 0

...