llvm/test/CodeGen/AMDGPU/sdwa-peephole-cndmask-wave64.mir

# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
# RUN: llc %s -mtriple=amdgcn -mcpu=gfx900 -run-pass=si-peephole-sdwa -o - | FileCheck %s

---
name: cndmask_b32 # can be directly converted to SDWA without a copy to VCC
tracksRegLiveness: true
body: |
  bb.0:
    liveins: $vgpr0, $vgpr1, $vcc

    ; CHECK-LABEL: name: cndmask_b32
    ; CHECK: liveins: $vgpr0, $vgpr1, $vcc
    ; CHECK-NEXT: {{  $}}
    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
    ; CHECK-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, [[COPY]], implicit $exec
    ; CHECK-NEXT: [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, [[COPY1]], implicit $exec
    ; CHECK-NEXT: [[V_CNDMASK_B32_sdwa:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_sdwa 0, [[COPY]], 0, [[COPY1]], 0, 6, 0, 5, 5, implicit $vcc, implicit $exec
    ; CHECK-NEXT: $vgpr0 = COPY [[V_CNDMASK_B32_sdwa]]
    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
    %0:vgpr_32 = COPY $vgpr1
    %1:vgpr_32 = COPY $vgpr0
    %2:vgpr_32 = V_LSHRREV_B32_e64 16, %0, implicit $exec
    %3:vgpr_32 = V_LSHRREV_B32_e64 16, %1, implicit $exec
    %4:vgpr_32 = V_CNDMASK_B32_e32 killed %2, killed %3, implicit $exec, implicit $vcc
    $vgpr0 = COPY %4
    SI_RETURN implicit $vgpr0

...

# For SDWA conversion of V_CNDMASK, the carry-in operand must be
# available in VCC.  This is achieved by introducing a COPY
# instruction. Comparison instructions could be changed to VOP2 form
# instead, but we prefer to use a COPY.

---
name: carry-compare
tracksRegLiveness: true
body:             |
  bb.0:
    liveins: $vgpr0

    ; CHECK-LABEL: name: carry-compare
    ; CHECK: liveins: $vgpr0
    ; CHECK-NEXT: {{  $}}
    ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
    ; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[DEF]], 1, implicit $exec
    ; CHECK-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, [[DEF1]], implicit $exec
    ; CHECK-NEXT: $vcc = COPY killed [[V_CMP_EQ_U32_e64_]]
    ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
    ; CHECK-NEXT: [[V_CNDMASK_B32_sdwa:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_sdwa 0, [[V_MOV_B32_e32_]], 0, [[DEF1]], 0, 6, 0, 6, 5, implicit $vcc, implicit $exec
    ; CHECK-NEXT: $vgpr0 = COPY [[V_CNDMASK_B32_sdwa]]
    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
    %0:vgpr_32 = IMPLICIT_DEF
    %1:vgpr_32 = IMPLICIT_DEF
    %2:sreg_64_xexec = V_CMP_EQ_U32_e64 %0, 1, implicit $exec
    %3:vgpr_32 = V_LSHRREV_B32_e64 16, %1, implicit $exec
    %4:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, %3, killed %2, implicit $exec
    $vgpr0 = COPY %4
    SI_RETURN implicit $vgpr0
...

---
name: carry-compare-class
tracksRegLiveness: true
body:             |
  bb.0:
    liveins: $vgpr0

    ; CHECK-LABEL: name: carry-compare-class
    ; CHECK: liveins: $vgpr0
    ; CHECK-NEXT: {{  $}}
    ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
    ; CHECK-NEXT: [[V_CMP_CLASS_F32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_CLASS_F32_e64 2, [[DEF]], 1, implicit $exec
    ; CHECK-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, [[DEF1]], implicit $exec
    ; CHECK-NEXT: $vcc = COPY killed [[V_CMP_CLASS_F32_e64_]]
    ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
    ; CHECK-NEXT: [[V_CNDMASK_B32_sdwa:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_sdwa 0, [[V_MOV_B32_e32_]], 0, [[DEF1]], 0, 6, 0, 6, 5, implicit $vcc, implicit $exec
    ; CHECK-NEXT: $vgpr0 = COPY [[V_CNDMASK_B32_sdwa]]
    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
    %0:sreg_32_xm0_xexec = IMPLICIT_DEF
    %1:vgpr_32 = IMPLICIT_DEF
    %2:sreg_64_xexec = V_CMP_CLASS_F32_e64 2,%0, 1, implicit $exec
    %3:vgpr_32 = V_LSHRREV_B32_e64 16, %1, implicit $exec
    %4:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, %3, killed %2, implicit $exec
    $vgpr0 = COPY %4
    SI_RETURN implicit $vgpr0
...

...
---
name: carry-non-compare
tracksRegLiveness: true
body:             |
  bb.0:
    liveins: $vgpr0

    ; CHECK-LABEL: name: carry-non-compare
    ; CHECK: liveins: $vgpr0
    ; CHECK-NEXT: {{  $}}
    ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
    ; CHECK-NEXT: [[V_PK_MAX_F16_:%[0-9]+]]:vgpr_32 = V_PK_MAX_F16 8, [[DEF1]], 8, [[DEF1]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
    ; CHECK-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, [[V_PK_MAX_F16_]], implicit $exec
    ; CHECK-NEXT: $vcc = COPY killed [[DEF]]
    ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
    ; CHECK-NEXT: [[V_CNDMASK_B32_sdwa:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_sdwa 0, [[V_MOV_B32_e32_]], 0, [[V_PK_MAX_F16_]], 0, 6, 0, 6, 5, implicit $vcc, implicit $exec
    ; CHECK-NEXT: $vgpr0 = COPY [[V_CNDMASK_B32_sdwa]]
    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
    %0:sreg_64_xexec = IMPLICIT_DEF
    %1:vgpr_32 = IMPLICIT_DEF
    %2:vgpr_32 = V_PK_MAX_F16 8, %1, 8, %1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
    %3:vgpr_32 = V_LSHRREV_B32_e64 16, %2, implicit $exec
    %4:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, %3, killed %0, implicit $exec
    $vgpr0 = COPY %4
    SI_RETURN implicit $vgpr0
...

...
---
name: carry-multiuse
tracksRegLiveness: true
body:             |
  bb.0:
    liveins: $vgpr0, $vgpr1

    ; CHECK-LABEL: name: carry-multiuse
    ; CHECK: liveins: $vgpr0, $vgpr1
    ; CHECK-NEXT: {{  $}}
    ; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
    ; CHECK-NEXT: [[V_PK_MAX_F16_:%[0-9]+]]:vgpr_32 = V_PK_MAX_F16 8, undef [[DEF1]], 8, undef [[DEF1]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
    ; CHECK-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, [[V_PK_MAX_F16_]], implicit $exec
    ; CHECK-NEXT: $vcc = COPY [[DEF]]
    ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
    ; CHECK-NEXT: [[V_CNDMASK_B32_sdwa:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_sdwa 0, [[V_MOV_B32_e32_]], 0, [[V_PK_MAX_F16_]], 0, 6, 0, 6, 5, implicit $vcc, implicit $exec
    ; CHECK-NEXT: $vgpr0 = COPY [[V_CNDMASK_B32_sdwa]]
    ; CHECK-NEXT: $vgpr1 = COPY [[DEF]]
    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
    %0:sreg_64_xexec = IMPLICIT_DEF
    %1:vgpr_32 = IMPLICIT_DEF
    %2:vgpr_32 = V_PK_MAX_F16 8, undef %1, 8, undef %1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
    %3:vgpr_32 = V_LSHRREV_B32_e64 16, %2, implicit $exec
    %4:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, %3, %0, implicit $exec
    $vgpr0 = COPY %4
    $vgpr1 = COPY %0
    SI_RETURN implicit $vgpr0
...

...
---
name: live-vcc # cannot convert because of live VCC
tracksRegLiveness: true
body:             |
  bb.0:
    liveins: $vgpr0

    ; CHECK-LABEL: name: live-vcc
    ; CHECK: liveins: $vgpr0
    ; CHECK-NEXT: {{  $}}
    ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
    ; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[DEF]], 1, implicit $exec
    ; CHECK-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, [[DEF1]], implicit $exec
    ; CHECK-NEXT: V_CMP_EQ_U32_e32 1, [[DEF1]], implicit-def $vcc, implicit $exec
    ; CHECK-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, [[V_LSHRREV_B32_e64_]], [[V_CMP_EQ_U32_e64_]], implicit $exec
    ; CHECK-NEXT: [[V_CNDMASK_B32_e32_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e32 killed [[V_LSHRREV_B32_e64_]], killed [[V_LSHRREV_B32_e64_]], implicit $vcc, implicit $exec
    ; CHECK-NEXT: $vgpr0 = COPY [[V_CNDMASK_B32_e64_]]
    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
    %0:vgpr_32 = IMPLICIT_DEF
    %2:vgpr_32 = IMPLICIT_DEF
    %3:sreg_64_xexec = V_CMP_EQ_U32_e64 %0, 1, implicit $exec
    %5:vgpr_32 = V_LSHRREV_B32_e64 16, %2, implicit $exec
    V_CMP_EQ_U32_e32 1, %2, implicit-def $vcc, implicit $exec
    %6:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, %5, %3, implicit $exec
    %7:vgpr_32 = V_CNDMASK_B32_e32 killed %5, killed %5, implicit $vcc, implicit $exec
    $vgpr0 = COPY %6
    SI_RETURN implicit $vgpr0
...

...
---
name: cannot-shrink-with-source-mods
tracksRegLiveness: true
body:             |
  bb.0:
    liveins: $vgpr0

    ; CHECK-LABEL: name: cannot-shrink-with-source-mods
    ; CHECK: liveins: $vgpr0
    ; CHECK-NEXT: {{  $}}
    ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
    ; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 undef [[DEF]], 0, implicit $exec
    ; CHECK-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, undef [[DEF1]], implicit $exec
    ; CHECK-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 1, 0, 0, [[V_LSHRREV_B32_e64_]], killed [[V_CMP_EQ_U32_e64_]], implicit $exec
    ; CHECK-NEXT: $vgpr0 = COPY [[V_CNDMASK_B32_e64_]]
    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
    %0:vgpr_32 = IMPLICIT_DEF
    %1:vgpr_32 = IMPLICIT_DEF
    %2:sreg_64_xexec = V_CMP_EQ_U32_e64 undef %0, 0, implicit $exec
    %3:vgpr_32 = V_LSHRREV_B32_e64 16, undef %1, implicit $exec
    %4:vgpr_32 = V_CNDMASK_B32_e64 1, 0, 0, %3, killed %2, implicit $exec
    $vgpr0 = COPY %4
    SI_RETURN implicit $vgpr0
...

...
---
name: missing-carry-def
tracksRegLiveness: true
registers:
  - { id: 0, class: sreg_64_xexec }
body:             |
  bb.0:
    liveins: $vgpr0

    ; CHECK-LABEL: name: missing-carry-def
    ; CHECK: liveins: $vgpr0
    ; CHECK-NEXT: {{  $}}
    ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
    ; CHECK-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, undef [[DEF]], implicit $exec
    ; CHECK-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, [[V_LSHRREV_B32_e64_]], undef %0:sreg_64_xexec, implicit $exec
    ; CHECK-NEXT: $vgpr0 = COPY [[V_CNDMASK_B32_e64_]]
    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
    %1:vgpr_32 = IMPLICIT_DEF
    %2:vgpr_32 = V_LSHRREV_B32_e64 16, undef %1, implicit $exec
    %3:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, %2, undef %0:sreg_64_xexec, implicit $exec
    $vgpr0 = COPY %3
    SI_RETURN implicit $vgpr0
...

# SDWA conversion of V_CNDMASK_B32 with V_BFE_I32 operand had to be
# disabled.
# FIXME sext modifier gets erroneously printed as neg modifier.

...
---
name: issue138766_cndmask_with_sext
tracksRegLiveness: true
body:             |
  bb.0:
    liveins: $vgpr0, $vgpr1, $vgpr2

    ; CHECK-LABEL: name: issue138766_cndmask_with_sext
    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2
    ; CHECK-NEXT: {{  $}}
    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr2
    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
    ; CHECK-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 1, [[COPY]], implicit $exec
    ; CHECK-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 killed [[V_AND_B32_e64_]], 1, implicit $exec
    ; CHECK-NEXT: [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[COPY2]], 0, 8, implicit $exec
    ; CHECK-NEXT: $vcc = COPY killed [[V_CMP_EQ_U32_e64_]]
    ; CHECK-NEXT: [[V_CNDMASK_B32_e32_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e32 0, killed [[V_BFE_I32_e64_]], implicit $vcc, implicit $exec
    ; CHECK-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 killed [[V_CNDMASK_B32_e32_]], [[COPY1]], implicit $exec
    ; CHECK-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
    ; CHECK-NEXT: SI_RETURN implicit $vgpr0
    %10:vgpr_32 = COPY $vgpr2
    %9:vgpr_32 = COPY $vgpr1
    %8:vgpr_32 = COPY $vgpr0
    %11:vgpr_32 = V_AND_B32_e64 1, %10, implicit $exec
    %12:sreg_64_xexec = V_CMP_EQ_U32_e64 killed %11, 1, implicit $exec
    %14:vgpr_32 = V_BFE_I32_e64 %8, 0, 8, implicit $exec
    %16:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, killed %14, killed %12, implicit $exec
    %18:vgpr_32 = V_OR_B32_e64 killed %16, %9, implicit $exec
    $vgpr0 = COPY %18
    SI_RETURN implicit $vgpr0
...