| # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 |
| # RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=pipeliner -pipeliner-force-ii=12 %s -o - | FileCheck %s |
| # RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -passes=pipeliner -pipeliner-force-ii=12 %s -o - | FileCheck %s |
| # Two independent recurrences (a sum and a product) must be carried across the stage |
| # boundary without being swapped: each accumulator's kernel result has to flow through |
| # its own epilog PHI and drain op into its own store. |
| |
| --- |
| name: swp_amdgpu_pipeline_epilog_phi |
| tracksRegLiveness: true |
| isSSA: true |
| liveins: |
| - { reg: '$sgpr4_sgpr5' } |
| machineFunctionInfo: |
| isEntryFunction: true |
| argumentInfo: |
| kernargSegmentPtr: { reg: '$sgpr4_sgpr5' } |
| body: | |
| ; CHECK-LABEL: name: swp_amdgpu_pipeline_epilog_phi |
| ; CHECK: bb.0: |
| ; CHECK-NEXT: successors: %bb.3(0x80000000) |
| ; CHECK-NEXT: liveins: $sgpr4_sgpr5 |
| ; CHECK-NEXT: {{ $}} |
| ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5 |
| ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]](p4), 0, 0 :: (dereferenceable invariant load (s128), addrspace 4) |
| ; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 16, 0 :: (dereferenceable invariant load (s32), align 16, addrspace 4) |
| ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE [[S_LOAD_DWORDX4_IMM]].sub0, %subreg.sub0, [[S_LOAD_DWORDX4_IMM]].sub1, %subreg.sub1 |
| ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE [[S_LOAD_DWORDX4_IMM]].sub2, %subreg.sub0, [[S_LOAD_DWORDX4_IMM]].sub3, %subreg.sub1 |
| ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0 |
| ; CHECK-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1065353216, implicit $exec |
| ; CHECK-NEXT: [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec |
| ; CHECK-NEXT: {{ $}} |
| ; CHECK-NEXT: bb.3: |
| ; CHECK-NEXT: successors: %bb.4(0x80000000), %bb.5(0x00000000) |
| ; CHECK-NEXT: {{ $}} |
| ; CHECK-NEXT: [[S_ASHR_I32_:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 [[S_MOV_B32_]], 31, implicit-def dead $scc |
| ; CHECK-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_ASHR_I32_]], %subreg.sub1 |
| ; CHECK-NEXT: [[S_LSHL_B64_:%[0-9]+]]:sreg_64 = S_LSHL_B64 [[REG_SEQUENCE2]], 2, implicit-def dead $scc |
| ; CHECK-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[REG_SEQUENCE]].sub0, [[S_LSHL_B64_]].sub0, implicit-def $scc |
| ; CHECK-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[REG_SEQUENCE]].sub1, [[S_LSHL_B64_]].sub1, implicit-def $scc, implicit $scc |
| ; CHECK-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1 |
| ; CHECK-NEXT: [[S_ADD_U32_1:%[0-9]+]]:sreg_32 = S_ADD_U32 [[REG_SEQUENCE1]].sub0, [[S_LSHL_B64_]].sub0, implicit-def $scc |
| ; CHECK-NEXT: [[S_ADDC_U32_1:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[REG_SEQUENCE1]].sub1, [[S_LSHL_B64_]].sub1, implicit-def $scc, implicit $scc |
| ; CHECK-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_1]], %subreg.sub0, [[S_ADDC_U32_1]], %subreg.sub1 |
| ; CHECK-NEXT: [[S_LOAD_DWORD_IMM1:%[0-9]+]]:sgpr_32 = S_LOAD_DWORD_IMM [[REG_SEQUENCE3]], 0, 0 :: (load (s32), addrspace 1) |
| ; CHECK-NEXT: [[S_LOAD_DWORD_IMM2:%[0-9]+]]:sgpr_32 = S_LOAD_DWORD_IMM [[REG_SEQUENCE4]], 0, 0 :: (load (s32), addrspace 1) |
| ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_MOV_B32_]], 1, implicit-def dead $scc |
| ; CHECK-NEXT: S_CMP_LT_I32 [[S_ADD_I32_]], [[S_LOAD_DWORD_IMM]], implicit-def $scc |
| ; CHECK-NEXT: S_CBRANCH_SCC0 %bb.5, implicit $scc |
| ; CHECK-NEXT: S_BRANCH %bb.4 |
| ; CHECK-NEXT: {{ $}} |
| ; CHECK-NEXT: bb.4: |
| ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.4(0x40000000) |
| ; CHECK-NEXT: {{ $}} |
| ; CHECK-NEXT: [[PHI:%[0-9]+]]:sreg_32 = PHI [[S_ADD_I32_]], %bb.3, %46, %bb.4 |
| ; CHECK-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[AV_MOV_1]], %bb.3, %51, %bb.4 |
| ; CHECK-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[AV_MOV_]], %bb.3, %49, %bb.4 |
| ; CHECK-NEXT: [[PHI3:%[0-9]+]]:sgpr_32 = PHI [[S_LOAD_DWORD_IMM1]], %bb.3, %50, %bb.4 |
| ; CHECK-NEXT: [[PHI4:%[0-9]+]]:sgpr_32 = PHI [[S_LOAD_DWORD_IMM2]], %bb.3, %48, %bb.4 |
| ; CHECK-NEXT: [[S_ASHR_I32_1:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 [[PHI]], 31, implicit-def dead $scc |
| ; CHECK-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[PHI]], %subreg.sub0, [[S_ASHR_I32_1]], %subreg.sub1 |
| ; CHECK-NEXT: [[S_LSHL_B64_1:%[0-9]+]]:sreg_64 = S_LSHL_B64 [[REG_SEQUENCE5]], 2, implicit-def dead $scc |
| ; CHECK-NEXT: [[S_ADD_U32_2:%[0-9]+]]:sreg_32 = S_ADD_U32 [[REG_SEQUENCE]].sub0, [[S_LSHL_B64_1]].sub0, implicit-def $scc |
| ; CHECK-NEXT: [[S_ADDC_U32_2:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[REG_SEQUENCE]].sub1, [[S_LSHL_B64_1]].sub1, implicit-def $scc, implicit $scc |
| ; CHECK-NEXT: [[S_ADD_U32_3:%[0-9]+]]:sreg_32 = S_ADD_U32 [[REG_SEQUENCE1]].sub0, [[S_LSHL_B64_1]].sub0, implicit-def $scc |
| ; CHECK-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_2]], %subreg.sub0, [[S_ADDC_U32_2]], %subreg.sub1 |
| ; CHECK-NEXT: [[S_ADDC_U32_3:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[REG_SEQUENCE1]].sub1, [[S_LSHL_B64_1]].sub1, implicit-def $scc, implicit $scc |
| ; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[PHI]], 1, implicit-def dead $scc |
| ; CHECK-NEXT: [[REG_SEQUENCE7:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_3]], %subreg.sub0, [[S_ADDC_U32_3]], %subreg.sub1 |
| ; CHECK-NEXT: [[S_LOAD_DWORD_IMM3:%[0-9]+]]:sgpr_32 = S_LOAD_DWORD_IMM [[REG_SEQUENCE7]], 0, 0 :: (load (s32), addrspace 1) |
| ; CHECK-NEXT: [[V_MUL_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_MUL_F32_e32 [[PHI4]], [[PHI2]], implicit $mode, implicit $exec |
| ; CHECK-NEXT: [[S_LOAD_DWORD_IMM4:%[0-9]+]]:sgpr_32 = S_LOAD_DWORD_IMM [[REG_SEQUENCE6]], 0, 0 :: (load (s32), addrspace 1) |
| ; CHECK-NEXT: [[V_ADD_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e32 [[PHI3]], [[PHI1]], implicit $mode, implicit $exec |
| ; CHECK-NEXT: S_CMP_LT_I32 [[S_ADD_I32_1]], [[S_LOAD_DWORD_IMM]], implicit-def $scc |
| ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.4, implicit $scc |
| ; CHECK-NEXT: S_BRANCH %bb.5 |
| ; CHECK-NEXT: {{ $}} |
| ; CHECK-NEXT: bb.5: |
| ; CHECK-NEXT: successors: %bb.2(0x80000000) |
| ; CHECK-NEXT: {{ $}} |
| ; CHECK-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI [[AV_MOV_1]], %bb.3, [[V_ADD_F32_e32_]], %bb.4 |
| ; CHECK-NEXT: [[PHI6:%[0-9]+]]:vgpr_32 = PHI [[AV_MOV_]], %bb.3, [[V_MUL_F32_e32_]], %bb.4 |
| ; CHECK-NEXT: [[PHI7:%[0-9]+]]:sgpr_32 = PHI [[S_LOAD_DWORD_IMM1]], %bb.3, [[S_LOAD_DWORD_IMM4]], %bb.4 |
| ; CHECK-NEXT: [[PHI8:%[0-9]+]]:sgpr_32 = PHI [[S_LOAD_DWORD_IMM2]], %bb.3, [[S_LOAD_DWORD_IMM3]], %bb.4 |
| ; CHECK-NEXT: [[V_ADD_F32_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e32 [[PHI7]], [[PHI5]], implicit $mode, implicit $exec |
| ; CHECK-NEXT: [[V_MUL_F32_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_MUL_F32_e32 [[PHI8]], [[PHI6]], implicit $mode, implicit $exec |
| ; CHECK-NEXT: S_BRANCH %bb.2 |
| ; CHECK-NEXT: {{ $}} |
| ; CHECK-NEXT: bb.2: |
| ; CHECK-NEXT: [[AV_MOV_2:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec |
| ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[AV_MOV_2]], [[V_ADD_F32_e32_1]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (store (s32), addrspace 1) |
| ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[AV_MOV_2]], [[V_MUL_F32_e32_1]], [[REG_SEQUENCE1]], 0, 0, implicit $exec :: (store (s32), addrspace 1) |
| ; CHECK-NEXT: S_ENDPGM 0 |
| bb.0: |
| liveins: $sgpr4_sgpr5 |
| |
| %0:sgpr_64(p4) = COPY $sgpr4_sgpr5 |
| %1:sgpr_128 = S_LOAD_DWORDX4_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s128), addrspace 4) |
| %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0(p4), 16, 0 :: (dereferenceable invariant load (s32), align 16, addrspace 4) |
| %3:sreg_64_xexec_xnull = REG_SEQUENCE %1.sub0, %subreg.sub0, %1.sub1, %subreg.sub1 |
| %4:sreg_64_xexec_xnull = REG_SEQUENCE %1.sub2, %subreg.sub0, %1.sub3, %subreg.sub1 |
| %5:sreg_32 = S_MOV_B32 0 |
| %6:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1065353216, implicit $exec |
| %7:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec |
| |
| bb.1: |
| successors: %bb.2, %bb.1 |
| |
| %8:sreg_32 = PHI %5, %bb.0, %9, %bb.1 |
| %10:vgpr_32 = PHI %7, %bb.0, %11, %bb.1 |
| %12:vgpr_32 = PHI %6, %bb.0, %13, %bb.1 |
| %14:sreg_32_xm0 = S_ASHR_I32 %8, 31, implicit-def dead $scc |
| %15:sreg_64 = REG_SEQUENCE %8, %subreg.sub0, %14, %subreg.sub1 |
| %16:sreg_64 = S_LSHL_B64 killed %15, 2, implicit-def dead $scc |
| %17:sreg_32 = S_ADD_U32 %3.sub0, %16.sub0, implicit-def $scc |
| %18:sreg_32 = S_ADDC_U32 %3.sub1, %16.sub1, implicit-def $scc, implicit $scc |
| %19:sreg_64 = REG_SEQUENCE %17, %subreg.sub0, %18, %subreg.sub1 |
| %20:sreg_32 = S_ADD_U32 %4.sub0, %16.sub0, implicit-def $scc |
| %21:sreg_32 = S_ADDC_U32 %4.sub1, %16.sub1, implicit-def $scc, implicit $scc |
| %22:sreg_64 = REG_SEQUENCE %20, %subreg.sub0, %21, %subreg.sub1 |
| %23:sgpr_32 = S_LOAD_DWORD_IMM killed %19, 0, 0 :: (load (s32), addrspace 1) |
| %24:sgpr_32 = S_LOAD_DWORD_IMM killed %22, 0, 0 :: (load (s32), addrspace 1) |
| %11:vgpr_32 = nofpexcept V_ADD_F32_e32 %23, %10, implicit $mode, implicit $exec |
| %13:vgpr_32 = nofpexcept V_MUL_F32_e32 %24, %12, implicit $mode, implicit $exec |
| %9:sreg_32 = S_ADD_I32 %8, 1, implicit-def dead $scc |
| S_CMP_LT_I32 %9, %2, implicit-def $scc |
| S_CBRANCH_SCC1 %bb.1, implicit $scc |
| S_BRANCH %bb.2 |
| |
| bb.2: |
| %25:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec |
| GLOBAL_STORE_DWORD_SADDR %25, %11, %3, 0, 0, implicit $exec :: (store (s32), addrspace 1) |
| GLOBAL_STORE_DWORD_SADDR %25, %13, %4, 0, 0, implicit $exec :: (store (s32), addrspace 1) |
| S_ENDPGM 0 |
| ... |
