| # RUN: llc -march=amdgcn -verify-machineinstrs -run-pass si-load-store-opt -o - %s | FileCheck %s |
| |
| # Check that SILoadStoreOptimizer honors memory dependencies between moved |
| # instructions. |
| # |
| # The following IR snippet would usually be optimized by the peephole optimizer. |
| # However, an equivalent situation can occur with buffer instructions as well. |
| |
| # CHECK-LABEL: name: mem_dependency |
| # CHECK: DS_READ2_B32 %0, 0, 1, |
| # CHECK: DS_WRITE_B32 %0, killed %1, 64, |
| # CHECK: DS_READ2_B32 %0, 16, 17, |
| # CHECK: DS_WRITE_B32 killed %0, %5, 0 |
| |
| --- | |
| define amdgpu_kernel void @mem_dependency(i32 addrspace(3)* %ptr.0) nounwind { |
| %ptr.4 = getelementptr i32, i32 addrspace(3)* %ptr.0, i32 1 |
| %ptr.64 = getelementptr i32, i32 addrspace(3)* %ptr.0, i32 16 |
| %1 = load i32, i32 addrspace(3)* %ptr.0 |
| store i32 %1, i32 addrspace(3)* %ptr.64 |
| %2 = load i32, i32 addrspace(3)* %ptr.64 |
| %3 = load i32, i32 addrspace(3)* %ptr.4 |
| %4 = add i32 %2, %3 |
| store i32 %4, i32 addrspace(3)* %ptr.0 |
| ret void |
| } |
| |
| @lds0 = external dso_local unnamed_addr addrspace(3) global [256 x i32], align 4 |
| @lds1 = external dso_local unnamed_addr addrspace(3) global [256 x i32], align 4 |
| @lds2 = external dso_local unnamed_addr addrspace(3) global [256 x i32], align 4 |
| @lds3 = external dso_local unnamed_addr addrspace(3) global [256 x i32], align 4 |
| |
| define void @asm_defines_address() #0 { |
| bb: |
| %tmp1 = load i32, i32 addrspace(3)* getelementptr inbounds ([256 x i32], [256 x i32] addrspace(3)* @lds0, i32 0, i32 0), align 4 |
| %0 = and i32 %tmp1, 255 |
| %tmp3 = load i32, i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds1, i32 0, i32 undef), align 4 |
| %tmp6 = load i32, i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds3, i32 0, i32 undef), align 4 |
| %tmp7 = tail call i32 asm "v_or_b32 $0, 0, $1", "=v,v"(i32 %tmp6) #1 |
| %tmp10 = lshr i32 %tmp7, 16 |
| %tmp11 = and i32 %tmp10, 255 |
| %tmp12 = getelementptr inbounds [256 x i32], [256 x i32] addrspace(3)* @lds1, i32 0, i32 %tmp11 |
| %tmp13 = load i32, i32 addrspace(3)* %tmp12, align 4 |
| %tmp14 = xor i32 %tmp3, %tmp13 |
| %tmp15 = lshr i32 %tmp14, 8 |
| %tmp16 = and i32 %tmp15, 16711680 |
| %tmp19 = lshr i32 %tmp16, 16 |
| %tmp20 = and i32 %tmp19, 255 |
| %tmp21 = getelementptr inbounds [256 x i32], [256 x i32] addrspace(3)* @lds1, i32 0, i32 %tmp20 |
| %tmp22 = load i32, i32 addrspace(3)* %tmp21, align 4 |
| %tmp24 = load i32, i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds2, i32 0, i32 undef), align 4 |
| %tmp25 = xor i32 %tmp22, %tmp24 |
| %tmp26 = and i32 %tmp25, -16777216 |
| %tmp28 = or i32 %0, %tmp26 |
| store volatile i32 %tmp28, i32 addrspace(1)* undef |
| ret void |
| } |
| |
| attributes #0 = { convergent nounwind } |
| attributes #1 = { convergent nounwind readnone } |
| |
| define amdgpu_kernel void @move_waw_hazards() #0 { |
| ret void |
| } |
| |
| attributes #0 = { convergent nounwind } |
| |
| define amdgpu_kernel void @merge_mmos(i32 addrspace(1)* %ptr_addr1) { ret void } |
| |
| ... |
| --- |
| name: mem_dependency |
| alignment: 1 |
| exposesReturnsTwice: false |
| legalized: false |
| regBankSelected: false |
| selected: false |
| tracksRegLiveness: true |
| liveins: |
| - { reg: '$vgpr0', virtual-reg: '%1' } |
| frameInfo: |
| isFrameAddressTaken: false |
| isReturnAddressTaken: false |
| hasStackMap: false |
| hasPatchPoint: false |
| stackSize: 0 |
| offsetAdjustment: 0 |
| maxAlignment: 0 |
| adjustsStack: false |
| hasCalls: false |
| maxCallFrameSize: 0 |
| hasOpaqueSPAdjustment: false |
| hasVAStart: false |
| hasMustTailInVarArgFunc: false |
| body: | |
| bb.0: |
| liveins: $vgpr0 |
| |
| %1:vgpr_32 = COPY $vgpr0 |
| $m0 = S_MOV_B32 -1 |
| %2:vgpr_32 = DS_READ_B32 %1, 0, 0, implicit $m0, implicit $exec :: (load 4 from %ir.ptr.0) |
| DS_WRITE_B32 %1, killed %2, 64, 0, implicit $m0, implicit $exec :: (store 4 into %ir.ptr.64) |
| |
| ; Make this load unmergeable, to tempt SILoadStoreOptimizer into merging the |
| ; other two loads. |
| %6:vreg_64 = DS_READ2_B32 %1, 16, 17, 0, implicit $m0, implicit $exec :: (load 8 from %ir.ptr.64, align 4) |
| %3:vgpr_32 = COPY %6.sub0 |
| %4:vgpr_32 = DS_READ_B32 %1, 4, 0, implicit $m0, implicit $exec :: (load 4 from %ir.ptr.4) |
| %5:vgpr_32 = V_ADD_I32_e32 killed %3, killed %4, implicit-def $vcc, implicit $exec |
| DS_WRITE_B32 killed %1, %5, 0, 0, implicit killed $m0, implicit $exec :: (store 4 into %ir.ptr.0) |
| S_ENDPGM 0 |
| |
| ... |
| --- |
| # Make sure the asm def isn't moved after the point where it's used for |
| # the address. |
| # CHECK-LABEL: name: asm_defines_address |
| # CHECK: DS_READ2ST64_B32 |
| # CHECK: DS_READ2ST64_B32 |
| # CHECK: INLINEASM |
| # CHECK: DS_READ_B32 |
| # CHECK: DS_READ_B32 |
| name: asm_defines_address |
| tracksRegLiveness: true |
| registers: |
| - { id: 0, class: vgpr_32, preferred-register: '' } |
| body: | |
| bb.0: |
| %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec |
| %2:vgpr_32 = DS_READ_B32 %1, 3072, 0, implicit $m0, implicit $exec :: (dereferenceable load 4 from `i32 addrspace(3)* getelementptr inbounds ([256 x i32], [256 x i32] addrspace(3)* @lds0, i32 0, i32 0)`, addrspace 3) |
| %3:vgpr_32 = DS_READ_B32 %1, 2048, 0, implicit $m0, implicit $exec :: (load 4 from `i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds1, i32 0, i32 undef)`, addrspace 3) |
| %4:vgpr_32 = DS_READ_B32 %1, 1024, 0, implicit $m0, implicit $exec :: (load 4 from `i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds3, i32 0, i32 undef)`, addrspace 3) |
| INLINEASM &"v_or_b32 $0, 0, $1", 32, 327690, def %0, 327689, %4 |
| %5:vgpr_32 = DS_READ_B32 %0, 2048, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp12, addrspace 3) |
| %6:vgpr_32 = DS_READ_B32 %5, 2048, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp21, addrspace 3) |
| %7:vgpr_32 = DS_READ_B32 %1, 0, 0, implicit $m0, implicit $exec :: (load 4 from `i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds2, i32 0, i32 undef)`, addrspace 3) |
| S_SETPC_B64_return undef $sgpr30_sgpr31, implicit %6, implicit %7 |
| |
| ... |
| --- |
| # Make sure Write-after-Write hazards are correctly detected and the |
| # instructions moved accordingly. |
| # operations. |
| # CHECK-LABEL: name: move_waw_hazards |
| # CHECK: S_AND_B64 |
| # CHECK: S_CMP_EQ_U32 |
| name: move_waw_hazards |
| tracksRegLiveness: true |
| body: | |
| bb.0: |
| liveins: $sgpr0_sgpr1 |
| |
| %3:sgpr_64 = COPY $sgpr0_sgpr1 |
| %6:sreg_32_xm0_xexec = S_MOV_B32 0 |
| %7:sreg_32_xm0 = S_MOV_B32 0 |
| %8:sreg_64_xexec = REG_SEQUENCE killed %6, %subreg.sub0, %7, %subreg.sub1 |
| %9:sgpr_128 = S_LOAD_DWORDX4_IMM killed %8, 0, 0, 0 :: (invariant load 16, addrspace 6) |
| %31:sreg_64_xexec = S_BUFFER_LOAD_DWORDX2_IMM %9, 0, 0, 0 :: (dereferenceable invariant load 4) |
| %10:sreg_32_xm0_xexec = COPY %31.sub0 |
| %11:sreg_32_xm0_xexec = COPY killed %31.sub1 |
| %12:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %9, 2, 0, 0 :: (dereferenceable invariant load 4) |
| %13:sreg_64 = V_CMP_NE_U32_e64 killed %11, 0, implicit $exec |
| %15:sreg_64 = V_CMP_NE_U32_e64 killed %12, 0, implicit $exec |
| %17:sreg_64_xexec = S_AND_B64 killed %13, killed %15, implicit-def dead $scc |
| S_CMP_EQ_U32 killed %10, 0, implicit-def $scc |
| %18:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %9, 3, 0, 0 :: (dereferenceable invariant load 4) |
| S_ENDPGM 0 |
| ... |
| --- |
| # CHECK-LABEL: merge_mmos |
| # CHECK: S_BUFFER_LOAD_DWORDX2_IMM %0, 0, 0, 0 :: (dereferenceable invariant load 8, align 4) |
| # CHECK: BUFFER_LOAD_DWORDX2_OFFSET %0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 4) |
| # CHECK: BUFFER_STORE_DWORDX2_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8, align 4) |
| # CHECK: BUFFER_LOAD_DWORDX2_OFFSET %0, 0, 64, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8 from %ir.ptr_addr1 + 64, align 4 |
| # CHECK: BUFFER_STORE_DWORDX2_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 64, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into %ir.ptr_addr1 + 64, align 4 |
| name: merge_mmos |
| tracksRegLiveness: true |
| body: | |
| bb.0: |
| liveins: $sgpr0_sgpr1_sgpr2_sgpr3 |
| |
| %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3 |
| %1:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %0, 0, 0, 0 :: (dereferenceable invariant load 4) |
| %2:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %0, 1, 0, 0 :: (dereferenceable invariant load 4) |
| %3:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4) |
| %4:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4) |
| BUFFER_STORE_DWORD_OFFSET_exact %3, %0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4) |
| BUFFER_STORE_DWORD_OFFSET_exact %4, %0, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4) |
| %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 64, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4 from %ir.ptr_addr1 + 64) |
| %6:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 68, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4 from %ir.ptr_addr1 + 68) |
| BUFFER_STORE_DWORD_OFFSET_exact %5, %0, 0, 64, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.ptr_addr1 + 64) |
| BUFFER_STORE_DWORD_OFFSET_exact %6, %0, 0, 68, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.ptr_addr1 + 68) |
| |
| S_ENDPGM 0 |
| |
| ... |