llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-128.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:atom\.)" --version 6
; RUN: llc -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx83 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,SPLIT
; RUN: llc -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx83 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,SPLIT
; RUN: llc -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx83 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,WIDE
; RUN: %if ptxas-sm_80 && ptxas-isa-8.3 %{ llc -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx83 %s -o - | %ptxas-verify -arch=sm_80 %}
; RUN: %if ptxas-sm_90 && ptxas-isa-8.3 %{ llc -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx83 %s -o - | %ptxas-verify -arch=sm_90 %}

; SM 70/80 support 128-bit atomic loads/stores, but not 128-bit RMWs.
; Elementwise RMWs must split even when the whole vector fits an atomic load.

define <2 x i64> @exchange128(ptr addrspace(1) %p, <2 x i64> %v) {
; SPLIT-LABEL: exchange128(
; SPLIT:    atom.relaxed.gpu.global.exch.b64 %rd4, [%rd1], %rd2;
; SPLIT:    atom.relaxed.gpu.global.exch.b64 %rd5, [%rd1+8], %rd3;
;
; WIDE-LABEL: exchange128(
; WIDE:    atom.relaxed.gpu.global.exch.b128 dst, [%rd1], amt;
  %r = atomicrmw elementwise xchg ptr addrspace(1) %p, <2 x i64> %v syncscope("device") monotonic, align 16
  ret <2 x i64> %r
}

define <2 x i64> @exchange128_align8(ptr addrspace(1) %p, <2 x i64> %v) {
; CHECK-LABEL: exchange128_align8(
; CHECK:    atom.relaxed.gpu.global.exch.b64 %rd4, [%rd1], %rd2;
; CHECK:    atom.relaxed.gpu.global.exch.b64 %rd5, [%rd1+8], %rd3;
  %r = atomicrmw elementwise xchg ptr addrspace(1) %p, <2 x i64> %v syncscope("device") monotonic, align 8
  ret <2 x i64> %r
}

; Apply the same RMW width limit when choosing a vector-wide CAS loop.
define <2 x double> @minimum128(ptr addrspace(1) %p, <2 x double> %v) {
; SPLIT-LABEL: minimum128(
; SPLIT:    atom.relaxed.gpu.global.cas.b64 %rd8, [%rd5], %rd3, %rd6;
; SPLIT:    atom.relaxed.gpu.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
;
; WIDE-LABEL: minimum128(
; WIDE:    atom.relaxed.gpu.global.cas.b128 dst, [%rd3], cmp, swap;
  %r = atomicrmw elementwise fmin ptr addrspace(1) %p, <2 x double> %v syncscope("device") monotonic, align 16
  ret <2 x double> %r
}