| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py |
| ; RUN: opt -S -mtriple=amdgcn-- -passes=amdgpu-atomic-optimizer %s | FileCheck -check-prefix=IR %s |
| |
| define amdgpu_kernel void @atomic_add_i32_offset(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_add_i32_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 4 |
| %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_add_i32_max_neg_offset(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_add_i32_max_neg_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 -1024 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 -1024 |
| %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_add_i32_soffset(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_add_i32_soffset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 9000 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 9000 |
| %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_add_i32_huge_offset(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_add_i32_huge_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 47224239175595 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 47224239175595 |
| |
| %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_add_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) { |
| ; IR-LABEL: @atomic_add_i32_ret_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ] |
| ; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP13]]) |
| ; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]] |
| ; IR-NEXT: [[TMP16:%.*]] = add i32 [[TMP14]], [[TMP15]] |
| ; IR-NEXT: store i32 [[TMP16]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 4 |
| %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_add_i32_addr64_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4 |
| %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_add_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_add_i32_ret_addr64_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ] |
| ; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP13]]) |
| ; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]] |
| ; IR-NEXT: [[TMP16:%.*]] = add i32 [[TMP14]], [[TMP15]] |
| ; IR-NEXT: store i32 [[TMP16]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4 |
| %val = atomicrmw volatile add ptr addrspace(1) %gep, i32 %in seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_add_i32(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_add_i32( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[OUT:%.*]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %val = atomicrmw volatile add ptr addrspace(1) %out, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_add_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) { |
| ; IR-LABEL: @atomic_add_i32_ret( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[OUT:%.*]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ] |
| ; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP13]]) |
| ; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]] |
| ; IR-NEXT: [[TMP16:%.*]] = add i32 [[TMP14]], [[TMP15]] |
| ; IR-NEXT: store i32 [[TMP16]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %val = atomicrmw volatile add ptr addrspace(1) %out, i32 %in seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_add_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_add_i32_addr64( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[PTR]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %val = atomicrmw volatile add ptr addrspace(1) %ptr, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_add_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_add_i32_ret_addr64( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile add ptr addrspace(1) [[PTR]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ] |
| ; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP13]]) |
| ; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]] |
| ; IR-NEXT: [[TMP16:%.*]] = add i32 [[TMP14]], [[TMP15]] |
| ; IR-NEXT: store i32 [[TMP16]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %val = atomicrmw volatile add ptr addrspace(1) %ptr, i32 %in seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_and_i32_offset(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_and_i32_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 4 |
| %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_and_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) { |
| ; IR-LABEL: @atomic_and_i32_ret_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = and i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP13]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 4 |
| %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_and_i32_addr64_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4 |
| %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_and_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_and_i32_ret_addr64_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = and i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP13]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4 |
| %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_and_i32(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_and_i32( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %val = atomicrmw volatile and ptr addrspace(1) %out, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_and_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) { |
| ; IR-LABEL: @atomic_and_i32_ret( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = and i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP13]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %val = atomicrmw volatile and ptr addrspace(1) %out, i32 %in seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_and_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_and_i32_addr64( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %val = atomicrmw volatile and ptr addrspace(1) %ptr, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_and_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_and_i32_ret_addr64( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile and ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = and i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP13]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %val = atomicrmw volatile and ptr addrspace(1) %ptr, i32 %in seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_sub_i32_offset(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_sub_i32_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 4 |
| %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_sub_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) { |
| ; IR-LABEL: @atomic_sub_i32_ret_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ] |
| ; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP13]]) |
| ; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]] |
| ; IR-NEXT: [[TMP16:%.*]] = sub i32 [[TMP14]], [[TMP15]] |
| ; IR-NEXT: store i32 [[TMP16]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 4 |
| %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_sub_i32_addr64_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4 |
| %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_sub_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_sub_i32_ret_addr64_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ] |
| ; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP13]]) |
| ; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]] |
| ; IR-NEXT: [[TMP16:%.*]] = sub i32 [[TMP14]], [[TMP15]] |
| ; IR-NEXT: store i32 [[TMP16]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4 |
| %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_sub_i32(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_sub_i32( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[OUT:%.*]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %val = atomicrmw volatile sub ptr addrspace(1) %out, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_sub_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) { |
| ; IR-LABEL: @atomic_sub_i32_ret( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[OUT:%.*]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ] |
| ; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP13]]) |
| ; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]] |
| ; IR-NEXT: [[TMP16:%.*]] = sub i32 [[TMP14]], [[TMP15]] |
| ; IR-NEXT: store i32 [[TMP16]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %val = atomicrmw volatile sub ptr addrspace(1) %out, i32 %in seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_sub_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_sub_i32_addr64( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[PTR]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %val = atomicrmw volatile sub ptr addrspace(1) %ptr, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_sub_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_sub_i32_ret_addr64( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]]) |
| ; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32 |
| ; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]] |
| ; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] |
| ; IR: 10: |
| ; IR-NEXT: [[TMP11:%.*]] = atomicrmw volatile sub ptr addrspace(1) [[PTR]], i32 [[TMP8]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP12]] |
| ; IR: 12: |
| ; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ] |
| ; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP13]]) |
| ; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]] |
| ; IR-NEXT: [[TMP16:%.*]] = sub i32 [[TMP14]], [[TMP15]] |
| ; IR-NEXT: store i32 [[TMP16]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %val = atomicrmw volatile sub ptr addrspace(1) %ptr, i32 %in seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_max_i32_offset(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_max_i32_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 4 |
| %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_max_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) { |
| ; IR-LABEL: @atomic_max_i32_ret_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 4 |
| %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_max_i32_addr64_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4 |
| %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_max_i32_ret_addr64_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4 |
| %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_max_i32(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_max_i32( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %val = atomicrmw volatile max ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_max_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) { |
| ; IR-LABEL: @atomic_max_i32_ret( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %val = atomicrmw volatile max ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_max_i32_addr64( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %val = atomicrmw volatile max ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_max_i32_ret_addr64( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile max ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %val = atomicrmw volatile max ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_umax_i32_offset(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_umax_i32_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 4 |
| %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_umax_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) { |
| ; IR-LABEL: @atomic_umax_i32_ret_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = icmp ugt i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 4 |
| %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_umax_i32_addr64_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4 |
| %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_umax_i32_ret_addr64_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = icmp ugt i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4 |
| %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_umax_i32(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_umax_i32( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %val = atomicrmw volatile umax ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_umax_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) { |
| ; IR-LABEL: @atomic_umax_i32_ret( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = icmp ugt i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %val = atomicrmw volatile umax ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_umax_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_umax_i32_addr64( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %val = atomicrmw volatile umax ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_umax_i32_ret_addr64( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile umax ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = icmp ugt i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %val = atomicrmw volatile umax ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_min_i32_offset(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_min_i32_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 4 |
| %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_min_i32_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) { |
| ; IR-LABEL: @atomic_min_i32_ret_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %gep = getelementptr i32, ptr addrspace(1) %out, i64 4 |
| %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_min_i32_addr64_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4 |
| %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_min_i32_ret_addr64_offset( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4 |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4 |
| %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) { |
| ; IR-LABEL: @atomic_min_i32( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %val = atomicrmw volatile min ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_min_i32_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in) { |
| ; IR-LABEL: @atomic_min_i32_ret( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %val = atomicrmw volatile min ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_min_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_min_i32_addr64( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %val = atomicrmw volatile min ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst |
| ret void |
| } |
| |
| define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i32 %in, i64 %index) { |
| ; IR-LABEL: @atomic_min_i32_ret_addr64( |
| ; IR-NEXT: entry: |
| ; IR-NEXT: [[PTR:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i64 [[INDEX:%.*]] |
| ; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) |
| ; IR-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP0]] to <2 x i32> |
| ; IR-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0 |
| ; IR-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1 |
| ; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0) |
| ; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]]) |
| ; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 |
| ; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]] |
| ; IR: 7: |
| ; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile min ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4 |
| ; IR-NEXT: br label [[TMP9]] |
| ; IR: 9: |
| ; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ] |
| ; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP10]]) |
| ; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN]] |
| ; IR-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP11]], [[TMP12]] |
| ; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]] |
| ; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4 |
| ; IR-NEXT: ret void |
| ; |
| entry: |
| %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index |
| %val = atomicrmw volatile min ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst |
| store i32 %val, ptr addrspace(1) %out2 |
| ret void |
| } |