llvm/test/CodeGen/AMDGPU/atomic-optimizer-volatile.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S -mtriple=amdgpu9.00-amd-amdhsa -passes='amdgpu-atomic-optimizer<strategy=iterative>,verify<domtree>' %s | FileCheck --check-prefixes=IR,IR-ITER %s
; RUN: opt -S -mtriple=amdgpu9.00-amd-amdhsa -passes='amdgpu-atomic-optimizer<strategy=dpp>,verify<domtree>' %s | FileCheck --check-prefixes=IR,IR-DPP %s

; Folding a wavefront into a single atomic would change the number of volatile
; operations, so these must be left alone.

define amdgpu_kernel void @volatile_global(ptr addrspace(1) %out, ptr addrspace(1) %inout) {
; IR-LABEL: define amdgpu_kernel void @volatile_global(
; IR-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[INOUT:%.*]]) {
; IR-NEXT:  [[ENTRY:.*:]]
; IR-NEXT:    [[OLD:%.*]] = atomicrmw volatile add ptr addrspace(1) [[INOUT]], i32 5 syncscope("agent") acq_rel, align 4
; IR-NEXT:    store i32 [[OLD]], ptr addrspace(1) [[OUT]], align 4
; IR-NEXT:    ret void
;
entry:
  %old = atomicrmw volatile add ptr addrspace(1) %inout, i32 5 syncscope("agent") acq_rel
  store i32 %old, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @volatile_local(ptr addrspace(1) %out, ptr addrspace(3) %inout) {
; IR-LABEL: define amdgpu_kernel void @volatile_local(
; IR-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(3) [[INOUT:%.*]]) {
; IR-NEXT:  [[ENTRY:.*:]]
; IR-NEXT:    [[OLD:%.*]] = atomicrmw volatile add ptr addrspace(3) [[INOUT]], i32 5 acq_rel, align 4
; IR-NEXT:    store i32 [[OLD]], ptr addrspace(1) [[OUT]], align 4
; IR-NEXT:    ret void
;
entry:
  %old = atomicrmw volatile add ptr addrspace(3) %inout, i32 5 acq_rel
  store i32 %old, ptr addrspace(1) %out
  ret void
}

; A divergent value is the only case where the iterative and DPP strategies
; differ, so cover it too.
define amdgpu_kernel void @volatile_global_divergent(ptr addrspace(1) %out, ptr addrspace(1) %inout) {
; IR-LABEL: define amdgpu_kernel void @volatile_global_divergent(
; IR-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[INOUT:%.*]]) {
; IR-NEXT:  [[ENTRY:.*:]]
; IR-NEXT:    [[ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
; IR-NEXT:    [[OLD:%.*]] = atomicrmw volatile add ptr addrspace(1) [[INOUT]], i32 [[ID]] syncscope("agent") acq_rel, align 4
; IR-NEXT:    store i32 [[OLD]], ptr addrspace(1) [[OUT]], align 4
; IR-NEXT:    ret void
;
entry:
  %id = call i32 @llvm.amdgcn.workitem.id.x()
  %old = atomicrmw volatile add ptr addrspace(1) %inout, i32 %id syncscope("agent") acq_rel
  store i32 %old, ptr addrspace(1) %out
  ret void
}

; Negative control: the same atomic without the volatile marker must be folded.
define amdgpu_kernel void @nonvolatile_global(ptr addrspace(1) %out, ptr addrspace(1) %inout) {
; IR-ITER-LABEL: define amdgpu_kernel void @nonvolatile_global(
; IR-ITER-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[INOUT:%.*]]) {
; IR-ITER-NEXT:  [[ENTRY:.*]]:
; IR-ITER-NEXT:    [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
; IR-ITER-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; IR-ITER-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
; IR-ITER-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-ITER-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-ITER-NEXT:    [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
; IR-ITER-NEXT:    [[TMP6:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 5, i32 1)
; IR-ITER-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-ITER-NEXT:    br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
; IR-ITER:       [[BB8]]:
; IR-ITER-NEXT:    [[TMP9:%.*]] = atomicrmw add ptr addrspace(1) [[INOUT]], i32 [[TMP6]] syncscope("agent") acq_rel, align 4
; IR-ITER-NEXT:    br label %[[BB10]]
; IR-ITER:       [[BB10]]:
; IR-ITER-NEXT:    [[TMP11:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[TMP9]], %[[BB8]] ]
; IR-ITER-NEXT:    [[TMP12:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP11]])
; IR-ITER-NEXT:    [[TMP13:%.*]] = mul i32 5, [[TMP5]]
; IR-ITER-NEXT:    [[TMP14:%.*]] = add i32 [[TMP12]], [[TMP13]]
; IR-ITER-NEXT:    store i32 [[TMP14]], ptr addrspace(1) [[OUT]], align 4
; IR-ITER-NEXT:    ret void
;
; IR-DPP-LABEL: define amdgpu_kernel void @nonvolatile_global(
; IR-DPP-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[INOUT:%.*]]) {
; IR-DPP-NEXT:  [[ENTRY:.*]]:
; IR-DPP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
; IR-DPP-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; IR-DPP-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
; IR-DPP-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-DPP-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-DPP-NEXT:    [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
; IR-DPP-NEXT:    [[TMP6:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 5, i32 2)
; IR-DPP-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-DPP-NEXT:    br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
; IR-DPP:       [[BB8]]:
; IR-DPP-NEXT:    [[TMP9:%.*]] = atomicrmw add ptr addrspace(1) [[INOUT]], i32 [[TMP6]] syncscope("agent") acq_rel, align 4
; IR-DPP-NEXT:    br label %[[BB10]]
; IR-DPP:       [[BB10]]:
; IR-DPP-NEXT:    [[TMP11:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[TMP9]], %[[BB8]] ]
; IR-DPP-NEXT:    [[TMP12:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP11]])
; IR-DPP-NEXT:    [[TMP13:%.*]] = mul i32 5, [[TMP5]]
; IR-DPP-NEXT:    [[TMP14:%.*]] = add i32 [[TMP12]], [[TMP13]]
; IR-DPP-NEXT:    store i32 [[TMP14]], ptr addrspace(1) [[OUT]], align 4
; IR-DPP-NEXT:    ret void
;
entry:
  %old = atomicrmw add ptr addrspace(1) %inout, i32 5 syncscope("agent") acq_rel
  store i32 %old, ptr addrspace(1) %out
  ret void
}

; cachepolicy bit 31 is the volatile flag for buffer atomics. Its operand index
; differs between the raw and struct forms, and between the rsrc and pointer
; forms, so cover all four.
define amdgpu_kernel void @volatile_raw_ptr_buffer(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; IR-LABEL: define amdgpu_kernel void @volatile_raw_ptr_buffer(
; IR-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(8) [[INOUT:%.*]]) {
; IR-NEXT:  [[ENTRY:.*:]]
; IR-NEXT:    [[OLD:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 5, ptr addrspace(8) [[INOUT]], i32 0, i32 0, i32 -2147483648)
; IR-NEXT:    store i32 [[OLD]], ptr addrspace(1) [[OUT]], align 4
; IR-NEXT:    ret void
;
entry:
  %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 5, ptr addrspace(8) %inout, i32 0, i32 0, i32 -2147483648)
  store i32 %old, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @volatile_struct_ptr_buffer(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; IR-LABEL: define amdgpu_kernel void @volatile_struct_ptr_buffer(
; IR-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(8) [[INOUT:%.*]]) {
; IR-NEXT:  [[ENTRY:.*:]]
; IR-NEXT:    [[OLD:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 5, ptr addrspace(8) [[INOUT]], i32 0, i32 0, i32 0, i32 -2147483648)
; IR-NEXT:    store i32 [[OLD]], ptr addrspace(1) [[OUT]], align 4
; IR-NEXT:    ret void
;
entry:
  %old = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add(i32 5, ptr addrspace(8) %inout, i32 0, i32 0, i32 0, i32 -2147483648)
  store i32 %old, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @volatile_raw_buffer(ptr addrspace(1) %out, <4 x i32> inreg %inout) {
; IR-LABEL: define amdgpu_kernel void @volatile_raw_buffer(
; IR-SAME: ptr addrspace(1) [[OUT:%.*]], <4 x i32> inreg [[INOUT:%.*]]) {
; IR-NEXT:  [[ENTRY:.*:]]
; IR-NEXT:    [[OLD:%.*]] = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 5, <4 x i32> [[INOUT]], i32 0, i32 0, i32 -2147483648)
; IR-NEXT:    store i32 [[OLD]], ptr addrspace(1) [[OUT]], align 4
; IR-NEXT:    ret void
;
entry:
  %old = call i32 @llvm.amdgcn.raw.buffer.atomic.add(i32 5, <4 x i32> %inout, i32 0, i32 0, i32 -2147483648)
  store i32 %old, ptr addrspace(1) %out
  ret void
}

define amdgpu_kernel void @volatile_struct_buffer(ptr addrspace(1) %out, <4 x i32> inreg %inout) {
; IR-LABEL: define amdgpu_kernel void @volatile_struct_buffer(
; IR-SAME: ptr addrspace(1) [[OUT:%.*]], <4 x i32> inreg [[INOUT:%.*]]) {
; IR-NEXT:  [[ENTRY:.*:]]
; IR-NEXT:    [[OLD:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 5, <4 x i32> [[INOUT]], i32 0, i32 0, i32 0, i32 -2147483648)
; IR-NEXT:    store i32 [[OLD]], ptr addrspace(1) [[OUT]], align 4
; IR-NEXT:    ret void
;
entry:
  %old = call i32 @llvm.amdgcn.struct.buffer.atomic.add(i32 5, <4 x i32> %inout, i32 0, i32 0, i32 0, i32 -2147483648)
  store i32 %old, ptr addrspace(1) %out
  ret void
}

; Negative control: only bit 31 means volatile, so a plain cachepolicy bit must
; not block the optimization.
define amdgpu_kernel void @nonvolatile_raw_ptr_buffer(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; IR-ITER-LABEL: define amdgpu_kernel void @nonvolatile_raw_ptr_buffer(
; IR-ITER-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(8) [[INOUT:%.*]]) {
; IR-ITER-NEXT:  [[ENTRY:.*]]:
; IR-ITER-NEXT:    [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
; IR-ITER-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; IR-ITER-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
; IR-ITER-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-ITER-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-ITER-NEXT:    [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
; IR-ITER-NEXT:    [[TMP6:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 5, i32 1)
; IR-ITER-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-ITER-NEXT:    br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
; IR-ITER:       [[BB8]]:
; IR-ITER-NEXT:    [[TMP9:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[TMP6]], ptr addrspace(8) [[INOUT]], i32 0, i32 0, i32 1)
; IR-ITER-NEXT:    br label %[[BB10]]
; IR-ITER:       [[BB10]]:
; IR-ITER-NEXT:    [[TMP11:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[TMP9]], %[[BB8]] ]
; IR-ITER-NEXT:    [[TMP12:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP11]])
; IR-ITER-NEXT:    [[TMP13:%.*]] = mul i32 5, [[TMP5]]
; IR-ITER-NEXT:    [[TMP14:%.*]] = add i32 [[TMP12]], [[TMP13]]
; IR-ITER-NEXT:    store i32 [[TMP14]], ptr addrspace(1) [[OUT]], align 4
; IR-ITER-NEXT:    ret void
;
; IR-DPP-LABEL: define amdgpu_kernel void @nonvolatile_raw_ptr_buffer(
; IR-DPP-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(8) [[INOUT:%.*]]) {
; IR-DPP-NEXT:  [[ENTRY:.*]]:
; IR-DPP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
; IR-DPP-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; IR-DPP-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
; IR-DPP-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-DPP-NEXT:    [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-DPP-NEXT:    [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
; IR-DPP-NEXT:    [[TMP6:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 5, i32 2)
; IR-DPP-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-DPP-NEXT:    br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
; IR-DPP:       [[BB8]]:
; IR-DPP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[TMP6]], ptr addrspace(8) [[INOUT]], i32 0, i32 0, i32 1)
; IR-DPP-NEXT:    br label %[[BB10]]
; IR-DPP:       [[BB10]]:
; IR-DPP-NEXT:    [[TMP11:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[TMP9]], %[[BB8]] ]
; IR-DPP-NEXT:    [[TMP12:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP11]])
; IR-DPP-NEXT:    [[TMP13:%.*]] = mul i32 5, [[TMP5]]
; IR-DPP-NEXT:    [[TMP14:%.*]] = add i32 [[TMP12]], [[TMP13]]
; IR-DPP-NEXT:    store i32 [[TMP14]], ptr addrspace(1) [[OUT]], align 4
; IR-DPP-NEXT:    ret void
;
entry:
  %old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 5, ptr addrspace(8) %inout, i32 0, i32 0, i32 1)
  store i32 %old, ptr addrspace(1) %out
  ret void
}