llvm/test/CodeGen/NVPTX/atomicrmw-elementwise-sm70.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ-DISALLOW
; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ-ALLOW
; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ-ALLOW
; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM70,SM70-FTZ-DISALLOW
; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM70,SM70-FTZ-ALLOW
; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM70,SM70-FTZ-ALLOW
; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_70 %}
; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_70 %}
; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | %ptxas-verify -arch=sm_70 %}
; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_70 %}
; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_70 %}
; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_70 %}

define <1 x i8> @xchg_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: xchg_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [xchg_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b8 %r5, [xchg_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r6, %rd2;
; SM70-NEXT:    and.b32 %r7, %r6, 3;
; SM70-NEXT:    shl.b32 %r1, %r7, 3;
; SM70-NEXT:    shl.b32 %r8, 255, %r1;
; SM70-NEXT:    not.b32 %r2, %r8;
; SM70-NEXT:    shl.b32 %r3, %r5, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-NEXT:  $L__BB0_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r9, %r12, %r2;
; SM70-NEXT:    or.b32 %r10, %r9, %r3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r12, %r10;
; SM70-NEXT:    setp.ne.b32 %p1, %r4, %r12;
; SM70-NEXT:    mov.b32 %r12, %r4;
; SM70-NEXT:    @%p1 bra $L__BB0_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r11, %r4, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r11;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @xchg_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: xchg_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [xchg_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    ld.param.b16 %r5, [xchg_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r6, %rd2;
; SM70-NEXT:    and.b32 %r7, %r6, 3;
; SM70-NEXT:    shl.b32 %r1, %r7, 3;
; SM70-NEXT:    shl.b32 %r8, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r8;
; SM70-NEXT:    shl.b32 %r3, %r5, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-NEXT:  $L__BB1_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r9, %r12, %r2;
; SM70-NEXT:    or.b32 %r10, %r9, %r3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r12, %r10;
; SM70-NEXT:    setp.ne.b32 %p1, %r4, %r12;
; SM70-NEXT:    mov.b32 %r12, %r4;
; SM70-NEXT:    @%p1 bra $L__BB1_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r11, %r4, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r11;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @xchg_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: xchg_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xchg_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    ld.param.b32 %r1, [xchg_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @xchg_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: xchg_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<4>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xchg_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    ld.param.b64 %rd2, [xchg_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @xchg_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: xchg_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [xchg_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %r5, [xchg_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r6, %rd2;
; SM70-NEXT:    and.b32 %r7, %r6, 3;
; SM70-NEXT:    shl.b32 %r1, %r7, 3;
; SM70-NEXT:    shl.b32 %r8, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r8;
; SM70-NEXT:    shl.b32 %r3, %r5, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-NEXT:  $L__BB4_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r9, %r12, %r2;
; SM70-NEXT:    or.b32 %r10, %r9, %r3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r12, %r10;
; SM70-NEXT:    setp.ne.b32 %p1, %r4, %r12;
; SM70-NEXT:    mov.b32 %r12, %r4;
; SM70-NEXT:    @%p1 bra $L__BB4_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r11, %r4, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r11;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @xchg_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: xchg_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xchg_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    ld.param.b32 %r1, [xchg_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @xchg_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: xchg_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<4>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xchg_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    ld.param.b64 %rd2, [xchg_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @xchg_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: xchg_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xchg_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    cvt.u64.u32 %rd2, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r2;
; SM70-NEXT:    shl.b64 %rd4, %rd3, 32;
; SM70-NEXT:    or.b64 %rd5, %rd2, %rd4;
; SM70-NEXT:    cvt.u64.u32 %rd6, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r4;
; SM70-NEXT:    shl.b64 %rd8, %rd7, 32;
; SM70-NEXT:    or.b64 %rd9, %rd6, %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd10, [%rd1], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @xchg_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: xchg_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xchg_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [xchg_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.exch.b32 %r2, [%rd1], %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @xchg_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: xchg_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<4>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xchg_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    ld.param.b64 %rd2, [xchg_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @xchg_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: xchg_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xchg_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    cvt.u64.u32 %rd2, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r2;
; SM70-NEXT:    shl.b64 %rd4, %rd3, 32;
; SM70-NEXT:    or.b64 %rd5, %rd2, %rd4;
; SM70-NEXT:    cvt.u64.u32 %rd6, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r4;
; SM70-NEXT:    shl.b64 %rd8, %rd7, 32;
; SM70-NEXT:    or.b64 %rd9, %rd6, %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd10, [%rd1], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @xchg_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: xchg_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xchg_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xchg_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xchg_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    cvt.u64.u32 %rd2, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r6;
; SM70-NEXT:    shl.b64 %rd4, %rd3, 32;
; SM70-NEXT:    or.b64 %rd5, %rd2, %rd4;
; SM70-NEXT:    cvt.u64.u32 %rd6, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r8;
; SM70-NEXT:    shl.b64 %rd8, %rd7, 32;
; SM70-NEXT:    or.b64 %rd9, %rd6, %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd10, [%rd1], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
; SM70-NEXT:    cvt.u64.u32 %rd12, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd13, %r2;
; SM70-NEXT:    shl.b64 %rd14, %rd13, 32;
; SM70-NEXT:    or.b64 %rd15, %rd12, %rd14;
; SM70-NEXT:    cvt.u64.u32 %rd16, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd17, %r4;
; SM70-NEXT:    shl.b64 %rd18, %rd17, 32;
; SM70-NEXT:    or.b64 %rd19, %rd16, %rd18;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd20, [%rd1+16], %rd15;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd21, [%rd1+24], %rd19;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd20, %rd21};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @xchg_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: xchg_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<4>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xchg_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd2, [xchg_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd3, [%rd1], %rd2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @xchg_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: xchg_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xchg_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd4, [%rd1], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd5, [%rd1+8], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd4, %rd5};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @xchg_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: xchg_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<10>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xchg_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd6, [%rd1], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd7, [%rd1+8], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd8, [%rd1+16], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd9, [%rd1+24], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @xchg_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: xchg_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<18>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xchg_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [xchg_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [xchg_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd6, %rd7}, [xchg_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [xchg_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd10, [%rd1], %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd11, [%rd1+8], %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd12, [%rd1+16], %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd13, [%rd1+24], %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd14, [%rd1+32], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd15, [%rd1+40], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd16, [%rd1+48], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.exch.b64 %rd17, [%rd1+56], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xchg ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @add_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: add_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [add_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b8 %r6, [add_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:    and.b32 %r8, %r7, 3;
; SM70-NEXT:    shl.b32 %r1, %r8, 3;
; SM70-NEXT:    shl.b32 %r2, 255, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    shl.b32 %r4, %r6, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:  $L__BB16_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    add.s32 %r9, %r14, %r4;
; SM70-NEXT:    and.b32 %r10, %r9, %r2;
; SM70-NEXT:    and.b32 %r11, %r14, %r3;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r14;
; SM70-NEXT:    mov.b32 %r14, %r5;
; SM70-NEXT:    @%p1 bra $L__BB16_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r5, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @add_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: add_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<12>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [add_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT:    ld.param.b64 %rd2, [add_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    and.b32 %r6, %r5, 3;
; SM70-NEXT:    shl.b32 %r1, %r6, 3;
; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r7;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
; SM70-NEXT:  $L__BB17_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r8, %r14, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r8;
; SM70-NEXT:    shr.u16 %rs4, %rs3, 8;
; SM70-NEXT:    add.s16 %rs7, %rs4, %rs6;
; SM70-NEXT:    add.s16 %rs8, %rs3, %rs5;
; SM70-NEXT:    and.b16 %rs9, %rs8, 255;
; SM70-NEXT:    shl.b16 %rs10, %rs7, 8;
; SM70-NEXT:    or.b16 %rs11, %rs9, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r9, %rs11;
; SM70-NEXT:    shl.b32 %r10, %r9, %r1;
; SM70-NEXT:    and.b32 %r11, %r14, %r2;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r14;
; SM70-NEXT:    mov.b32 %r14, %r3;
; SM70-NEXT:    @%p1 bra $L__BB17_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @add_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: add_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<19>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [add_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [add_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
; SM70-NEXT:  $L__BB18_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
; SM70-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
; SM70-NEXT:    add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
; SM70-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
; SM70-NEXT:    add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
; SM70-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
; SM70-NEXT:    add.s16 %rs12, %rs11, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r15, %rs12;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r18;
; SM70-NEXT:    mov.b32 %r18, %r1;
; SM70-NEXT:    @%p1 bra $L__BB18_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @add_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: add_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<35>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [add_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NEXT:  $L__BB19_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r3, %r1, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
; SM70-NEXT:    prmt.b32 %r6, %r1, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
; SM70-NEXT:    prmt.b32 %r7, %r33, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
; SM70-NEXT:    add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
; SM70-NEXT:    prmt.b32 %r10, %r1, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
; SM70-NEXT:    prmt.b32 %r11, %r33, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
; SM70-NEXT:    add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
; SM70-NEXT:    prmt.b32 %r13, %r1, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
; SM70-NEXT:    prmt.b32 %r14, %r33, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
; SM70-NEXT:    add.s16 %rs12, %rs11, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r15, %rs12;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r17;
; SM70-NEXT:    prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r18;
; SM70-NEXT:    prmt.b32 %r19, %r34, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs14, %r19;
; SM70-NEXT:    add.s16 %rs15, %rs14, %rs13;
; SM70-NEXT:    cvt.u32.u16 %r20, %rs15;
; SM70-NEXT:    prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs16, %r21;
; SM70-NEXT:    prmt.b32 %r22, %r34, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs17, %r22;
; SM70-NEXT:    add.s16 %rs18, %rs17, %rs16;
; SM70-NEXT:    cvt.u32.u16 %r23, %rs18;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r20, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs19, %r25;
; SM70-NEXT:    prmt.b32 %r26, %r34, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs20, %r26;
; SM70-NEXT:    add.s16 %rs21, %rs20, %rs19;
; SM70-NEXT:    cvt.u32.u16 %r27, %rs21;
; SM70-NEXT:    prmt.b32 %r28, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs22, %r28;
; SM70-NEXT:    prmt.b32 %r29, %r34, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs23, %r29;
; SM70-NEXT:    add.s16 %rs24, %rs23, %rs22;
; SM70-NEXT:    cvt.u32.u16 %r30, %rs24;
; SM70-NEXT:    prmt.b32 %r31, %r30, %r27, 0x3340U;
; SM70-NEXT:    prmt.b32 %r32, %r31, %r24, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r32;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB19_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @add_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: add_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [add_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %r6, [add_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:    and.b32 %r8, %r7, 3;
; SM70-NEXT:    shl.b32 %r1, %r8, 3;
; SM70-NEXT:    shl.b32 %r2, 65535, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    shl.b32 %r4, %r6, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:  $L__BB20_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    add.s32 %r9, %r14, %r4;
; SM70-NEXT:    and.b32 %r10, %r9, %r2;
; SM70-NEXT:    and.b32 %r11, %r14, %r3;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r14;
; SM70-NEXT:    mov.b32 %r14, %r5;
; SM70-NEXT:    @%p1 bra $L__BB20_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r5, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @add_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: add_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<7>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [add_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [add_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT:  $L__BB21_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT:    add.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    add.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p1 bra $L__BB21_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @add_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: add_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [add_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT:  $L__BB22_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-NEXT:    add.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    add.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
; SM70-NEXT:    add.s16 %rs11, %rs10, %rs8;
; SM70-NEXT:    add.s16 %rs12, %rs9, %rs7;
; SM70-NEXT:    mov.b32 %r4, {%rs12, %rs11};
; SM70-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB22_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @add_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: add_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT:  $L__BB23_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-NEXT:    add.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    add.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r5, {%rs6, %rs5};
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
; SM70-NEXT:    add.s16 %rs11, %rs10, %rs8;
; SM70-NEXT:    add.s16 %rs12, %rs9, %rs7;
; SM70-NEXT:    mov.b32 %r6, {%rs12, %rs11};
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB23_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-NEXT:  $L__BB23_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs15, %rs16}, %r11;
; SM70-NEXT:    add.s16 %rs17, %rs16, %rs14;
; SM70-NEXT:    add.s16 %rs18, %rs15, %rs13;
; SM70-NEXT:    mov.b32 %r7, {%rs18, %rs17};
; SM70-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
; SM70-NEXT:    add.s16 %rs23, %rs22, %rs20;
; SM70-NEXT:    add.s16 %rs24, %rs21, %rs19;
; SM70-NEXT:    mov.b32 %r8, {%rs24, %rs23};
; SM70-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NEXT:    @%p2 bra $L__BB23_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @add_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: add_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [add_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @add_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: add_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [add_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @add_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: add_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @add_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: add_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @add_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: add_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<4>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd2, [add_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd3, [%rd1], %rd2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @add_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: add_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd5, [%rd1+8], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd4, %rd5};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @add_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: add_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<10>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd6, [%rd1], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd8, [%rd1+16], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd9, [%rd1+24], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @add_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: add_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<18>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [add_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [add_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd6, %rd7}, [add_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [add_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd10, [%rd1], %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd11, [%rd1+8], %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd12, [%rd1+16], %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd14, [%rd1+32], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd15, [%rd1+40], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd16, [%rd1+48], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd17, [%rd1+56], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @sub_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: sub_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [sub_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b8 %r6, [sub_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:    and.b32 %r8, %r7, 3;
; SM70-NEXT:    shl.b32 %r1, %r8, 3;
; SM70-NEXT:    shl.b32 %r2, 255, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    shl.b32 %r4, %r6, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:  $L__BB32_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    sub.s32 %r9, %r14, %r4;
; SM70-NEXT:    and.b32 %r10, %r9, %r2;
; SM70-NEXT:    and.b32 %r11, %r14, %r3;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r14;
; SM70-NEXT:    mov.b32 %r14, %r5;
; SM70-NEXT:    @%p1 bra $L__BB32_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r5, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @sub_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: sub_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<12>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [sub_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT:    ld.param.b64 %rd2, [sub_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    and.b32 %r6, %r5, 3;
; SM70-NEXT:    shl.b32 %r1, %r6, 3;
; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r7;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
; SM70-NEXT:  $L__BB33_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r8, %r14, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r8;
; SM70-NEXT:    shr.u16 %rs4, %rs3, 8;
; SM70-NEXT:    sub.s16 %rs7, %rs4, %rs6;
; SM70-NEXT:    sub.s16 %rs8, %rs3, %rs5;
; SM70-NEXT:    and.b16 %rs9, %rs8, 255;
; SM70-NEXT:    shl.b16 %rs10, %rs7, 8;
; SM70-NEXT:    or.b16 %rs11, %rs9, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r9, %rs11;
; SM70-NEXT:    shl.b32 %r10, %r9, %r1;
; SM70-NEXT:    and.b32 %r11, %r14, %r2;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r14;
; SM70-NEXT:    mov.b32 %r14, %r3;
; SM70-NEXT:    @%p1 bra $L__BB33_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @sub_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: sub_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<19>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [sub_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [sub_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
; SM70-NEXT:  $L__BB34_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    sub.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
; SM70-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
; SM70-NEXT:    sub.s16 %rs6, %rs5, %rs4;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
; SM70-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
; SM70-NEXT:    sub.s16 %rs9, %rs8, %rs7;
; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
; SM70-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
; SM70-NEXT:    sub.s16 %rs12, %rs11, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r15, %rs12;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r18;
; SM70-NEXT:    mov.b32 %r18, %r1;
; SM70-NEXT:    @%p1 bra $L__BB34_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @sub_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: sub_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<35>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [sub_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NEXT:  $L__BB35_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r3, %r1, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    sub.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
; SM70-NEXT:    prmt.b32 %r6, %r1, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
; SM70-NEXT:    prmt.b32 %r7, %r33, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
; SM70-NEXT:    sub.s16 %rs6, %rs5, %rs4;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
; SM70-NEXT:    prmt.b32 %r10, %r1, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
; SM70-NEXT:    prmt.b32 %r11, %r33, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
; SM70-NEXT:    sub.s16 %rs9, %rs8, %rs7;
; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
; SM70-NEXT:    prmt.b32 %r13, %r1, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
; SM70-NEXT:    prmt.b32 %r14, %r33, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
; SM70-NEXT:    sub.s16 %rs12, %rs11, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r15, %rs12;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r17;
; SM70-NEXT:    prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r18;
; SM70-NEXT:    prmt.b32 %r19, %r34, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs14, %r19;
; SM70-NEXT:    sub.s16 %rs15, %rs14, %rs13;
; SM70-NEXT:    cvt.u32.u16 %r20, %rs15;
; SM70-NEXT:    prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs16, %r21;
; SM70-NEXT:    prmt.b32 %r22, %r34, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs17, %r22;
; SM70-NEXT:    sub.s16 %rs18, %rs17, %rs16;
; SM70-NEXT:    cvt.u32.u16 %r23, %rs18;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r20, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs19, %r25;
; SM70-NEXT:    prmt.b32 %r26, %r34, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs20, %r26;
; SM70-NEXT:    sub.s16 %rs21, %rs20, %rs19;
; SM70-NEXT:    cvt.u32.u16 %r27, %rs21;
; SM70-NEXT:    prmt.b32 %r28, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs22, %r28;
; SM70-NEXT:    prmt.b32 %r29, %r34, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs23, %r29;
; SM70-NEXT:    sub.s16 %rs24, %rs23, %rs22;
; SM70-NEXT:    cvt.u32.u16 %r30, %rs24;
; SM70-NEXT:    prmt.b32 %r31, %r30, %r27, 0x3340U;
; SM70-NEXT:    prmt.b32 %r32, %r31, %r24, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r32;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB35_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @sub_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: sub_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [sub_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %r6, [sub_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:    and.b32 %r8, %r7, 3;
; SM70-NEXT:    shl.b32 %r1, %r8, 3;
; SM70-NEXT:    shl.b32 %r2, 65535, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    shl.b32 %r4, %r6, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:  $L__BB36_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    sub.s32 %r9, %r14, %r4;
; SM70-NEXT:    and.b32 %r10, %r9, %r2;
; SM70-NEXT:    and.b32 %r11, %r14, %r3;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r14;
; SM70-NEXT:    mov.b32 %r14, %r5;
; SM70-NEXT:    @%p1 bra $L__BB36_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r5, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @sub_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: sub_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<7>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [sub_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [sub_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT:  $L__BB37_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT:    sub.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    sub.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p1 bra $L__BB37_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @sub_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: sub_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [sub_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT:  $L__BB38_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-NEXT:    sub.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    sub.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
; SM70-NEXT:    sub.s16 %rs11, %rs10, %rs8;
; SM70-NEXT:    sub.s16 %rs12, %rs9, %rs7;
; SM70-NEXT:    mov.b32 %r4, {%rs12, %rs11};
; SM70-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB38_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @sub_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: sub_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [sub_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT:  $L__BB39_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-NEXT:    sub.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    sub.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r5, {%rs6, %rs5};
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
; SM70-NEXT:    sub.s16 %rs11, %rs10, %rs8;
; SM70-NEXT:    sub.s16 %rs12, %rs9, %rs7;
; SM70-NEXT:    mov.b32 %r6, {%rs12, %rs11};
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB39_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-NEXT:  $L__BB39_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs15, %rs16}, %r11;
; SM70-NEXT:    sub.s16 %rs17, %rs16, %rs14;
; SM70-NEXT:    sub.s16 %rs18, %rs15, %rs13;
; SM70-NEXT:    mov.b32 %r7, {%rs18, %rs17};
; SM70-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
; SM70-NEXT:    sub.s16 %rs23, %rs22, %rs20;
; SM70-NEXT:    sub.s16 %rs24, %rs21, %rs19;
; SM70-NEXT:    mov.b32 %r8, {%rs24, %rs23};
; SM70-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NEXT:    @%p2 bra $L__BB39_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @sub_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: sub_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<4>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [sub_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [sub_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    neg.s32 %r2, %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @sub_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: sub_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [sub_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [sub_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    neg.s32 %r3, %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r4, [%rd1], %r3;
; SM70-NEXT:    neg.s32 %r5, %r2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r5;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @sub_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: sub_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [sub_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    neg.s32 %r5, %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r6, [%rd1], %r5;
; SM70-NEXT:    neg.s32 %r7, %r2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r8, [%rd1+4], %r7;
; SM70-NEXT:    neg.s32 %r9, %r3;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r10, [%rd1+8], %r9;
; SM70-NEXT:    neg.s32 %r11, %r4;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r11;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @sub_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: sub_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<25>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [sub_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [sub_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [sub_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    neg.s32 %r9, %r5;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r10, [%rd1], %r9;
; SM70-NEXT:    neg.s32 %r11, %r6;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r12, [%rd1+4], %r11;
; SM70-NEXT:    neg.s32 %r13, %r7;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r14, [%rd1+8], %r13;
; SM70-NEXT:    neg.s32 %r15, %r8;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r16, [%rd1+12], %r15;
; SM70-NEXT:    neg.s32 %r17, %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r18, [%rd1+16], %r17;
; SM70-NEXT:    neg.s32 %r19, %r2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r20, [%rd1+20], %r19;
; SM70-NEXT:    neg.s32 %r21, %r3;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r22, [%rd1+24], %r21;
; SM70-NEXT:    neg.s32 %r23, %r4;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r24, [%rd1+28], %r23;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @sub_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: sub_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<5>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [sub_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd2, [sub_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    neg.s64 %rd3, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd4, [%rd1], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd4;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @sub_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: sub_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<8>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [sub_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    neg.s64 %rd4, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd5, [%rd1], %rd4;
; SM70-NEXT:    neg.s64 %rd6, %rd3;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd7, [%rd1+8], %rd6;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd5, %rd7};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @sub_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: sub_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<14>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [sub_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    neg.s64 %rd6, %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd7, [%rd1], %rd6;
; SM70-NEXT:    neg.s64 %rd8, %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd9, [%rd1+8], %rd8;
; SM70-NEXT:    neg.s64 %rd10, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd11, [%rd1+16], %rd10;
; SM70-NEXT:    neg.s64 %rd12, %rd3;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd13, [%rd1+24], %rd12;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd11, %rd13};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd7, %rd9};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @sub_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: sub_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<26>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [sub_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [sub_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [sub_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd6, %rd7}, [sub_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [sub_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    neg.s64 %rd10, %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd11, [%rd1], %rd10;
; SM70-NEXT:    neg.s64 %rd12, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd13, [%rd1+8], %rd12;
; SM70-NEXT:    neg.s64 %rd14, %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd15, [%rd1+16], %rd14;
; SM70-NEXT:    neg.s64 %rd16, %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd17, [%rd1+24], %rd16;
; SM70-NEXT:    neg.s64 %rd18, %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd19, [%rd1+32], %rd18;
; SM70-NEXT:    neg.s64 %rd20, %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd21, [%rd1+40], %rd20;
; SM70-NEXT:    neg.s64 %rd22, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd23, [%rd1+48], %rd22;
; SM70-NEXT:    neg.s64 %rd24, %rd3;
; SM70-NEXT:    atom.relaxed.cta.global.add.u64 %rd25, [%rd1+56], %rd24;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd23, %rd25};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd19, %rd21};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd15, %rd17};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd11, %rd13};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise sub ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @and_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: and_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<11>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b8 %r1, [and_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd2, %rd1, -4;
; SM70-NEXT:    cvt.u32.u64 %r2, %rd1;
; SM70-NEXT:    and.b32 %r3, %r2, 3;
; SM70-NEXT:    shl.b32 %r4, %r3, 3;
; SM70-NEXT:    shl.b32 %r5, 255, %r4;
; SM70-NEXT:    not.b32 %r6, %r5;
; SM70-NEXT:    shl.b32 %r7, %r1, %r4;
; SM70-NEXT:    or.b32 %r8, %r7, %r6;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r9, [%rd2], %r8;
; SM70-NEXT:    shr.u32 %r10, %r9, %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r10;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @and_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: and_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<11>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd2, %rd1, -4;
; SM70-NEXT:    cvt.u32.u64 %r1, %rd1;
; SM70-NEXT:    and.b32 %r2, %r1, 3;
; SM70-NEXT:    shl.b32 %r3, %r2, 3;
; SM70-NEXT:    shl.b32 %r4, 65535, %r3;
; SM70-NEXT:    not.b32 %r5, %r4;
; SM70-NEXT:    ld.param.b16 %r6, [and_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    shl.b32 %r7, %r6, %r3;
; SM70-NEXT:    or.b32 %r8, %r7, %r5;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r9, [%rd2], %r8;
; SM70-NEXT:    shr.u32 %r10, %r9, %r3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r10;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @and_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: and_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [and_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:  $L__BB50_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r3, %r4, %r2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p1 bra $L__BB50_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @and_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: and_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:  $L__BB51_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r3, %r5, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NEXT:    and.b32 %r4, %r6, %r2;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB51_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @and_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: and_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<11>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %r1, [and_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd2, %rd1, -4;
; SM70-NEXT:    cvt.u32.u64 %r2, %rd1;
; SM70-NEXT:    and.b32 %r3, %r2, 3;
; SM70-NEXT:    shl.b32 %r4, %r3, 3;
; SM70-NEXT:    shl.b32 %r5, 65535, %r4;
; SM70-NEXT:    not.b32 %r6, %r5;
; SM70-NEXT:    shl.b32 %r7, %r1, %r4;
; SM70-NEXT:    or.b32 %r8, %r7, %r6;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r9, [%rd2], %r8;
; SM70-NEXT:    shr.u32 %r10, %r9, %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r10;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @and_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: and_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [and_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:  $L__BB53_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r3, %r4, %r2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p1 bra $L__BB53_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @and_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: and_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:  $L__BB54_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r3, %r5, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NEXT:    and.b32 %r4, %r6, %r2;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB54_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @and_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: and_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:  $L__BB55_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r5, %r9, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    and.b32 %r6, %r10, %r2;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB55_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NEXT:  $L__BB55_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r7, %r11, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NEXT:    and.b32 %r8, %r12, %r4;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NEXT:    @%p2 bra $L__BB55_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @and_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: and_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [and_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r2, [%rd1], %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @and_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: and_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [and_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r3, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r4, [%rd1+4], %r2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @and_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: and_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r5, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r6, [%rd1+4], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r7, [%rd1+8], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r8, [%rd1+12], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @and_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: and_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [and_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [and_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r9, [%rd1], %r5;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r10, [%rd1+4], %r6;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r11, [%rd1+8], %r7;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r12, [%rd1+12], %r8;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r13, [%rd1+16], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r14, [%rd1+20], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r15, [%rd1+24], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.and.b32 %r16, [%rd1+28], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @and_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: and_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<4>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd2, [and_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd3, [%rd1], %rd2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @and_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: and_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd4, [%rd1], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd5, [%rd1+8], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd4, %rd5};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @and_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: and_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<10>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd6, [%rd1], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd7, [%rd1+8], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd8, [%rd1+16], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd9, [%rd1+24], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @and_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: and_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<18>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [and_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [and_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [and_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd6, %rd7}, [and_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [and_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd10, [%rd1], %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd11, [%rd1+8], %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd12, [%rd1+16], %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd13, [%rd1+24], %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd14, [%rd1+32], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd15, [%rd1+40], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd16, [%rd1+48], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.and.b64 %rd17, [%rd1+56], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise and ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @nand_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: nand_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<16>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [nand_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b8 %r6, [nand_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:    and.b32 %r8, %r7, 3;
; SM70-NEXT:    shl.b32 %r1, %r8, 3;
; SM70-NEXT:    shl.b32 %r2, 255, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    shl.b32 %r4, %r6, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT:  $L__BB64_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r9, %r15, %r4;
; SM70-NEXT:    not.b32 %r10, %r9;
; SM70-NEXT:    and.b32 %r11, %r10, %r2;
; SM70-NEXT:    and.b32 %r12, %r15, %r3;
; SM70-NEXT:    or.b32 %r13, %r12, %r11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r15;
; SM70-NEXT:    mov.b32 %r15, %r5;
; SM70-NEXT:    @%p1 bra $L__BB64_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r14, %r5, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r14;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @nand_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: nand_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<16>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [nand_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r6, %rd2;
; SM70-NEXT:    and.b32 %r7, %r6, 3;
; SM70-NEXT:    shl.b32 %r1, %r7, 3;
; SM70-NEXT:    shl.b32 %r2, 65535, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    ld.param.b16 %r8, [nand_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    shl.b32 %r4, %r8, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT:  $L__BB65_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r9, %r15, %r4;
; SM70-NEXT:    not.b32 %r10, %r9;
; SM70-NEXT:    and.b32 %r11, %r10, %r2;
; SM70-NEXT:    and.b32 %r12, %r15, %r3;
; SM70-NEXT:    or.b32 %r13, %r12, %r11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r15;
; SM70-NEXT:    mov.b32 %r15, %r5;
; SM70-NEXT:    @%p1 bra $L__BB65_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r14, %r5, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r14;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @nand_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: nand_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<6>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [nand_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT:  $L__BB66_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r3, %r5, %r2;
; SM70-NEXT:    xor.b32 %r4, %r3, -1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r5;
; SM70-NEXT:    mov.b32 %r5, %r1;
; SM70-NEXT:    @%p1 bra $L__BB66_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @nand_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: nand_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:  $L__BB67_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r8;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r3, %r8, %r2;
; SM70-NEXT:    and.b32 %r4, %r7, %r1;
; SM70-NEXT:    xor.b32 %r5, %r4, -1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    xor.b32 %r6, %r3, -1;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB67_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @nand_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: nand_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<16>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [nand_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %r6, [nand_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:    and.b32 %r8, %r7, 3;
; SM70-NEXT:    shl.b32 %r1, %r8, 3;
; SM70-NEXT:    shl.b32 %r2, 65535, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    shl.b32 %r4, %r6, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT:  $L__BB68_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r9, %r15, %r4;
; SM70-NEXT:    not.b32 %r10, %r9;
; SM70-NEXT:    and.b32 %r11, %r10, %r2;
; SM70-NEXT:    and.b32 %r12, %r15, %r3;
; SM70-NEXT:    or.b32 %r13, %r12, %r11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r15;
; SM70-NEXT:    mov.b32 %r15, %r5;
; SM70-NEXT:    @%p1 bra $L__BB68_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r14, %r5, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r14;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @nand_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: nand_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<6>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [nand_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT:  $L__BB69_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r3, %r5, %r2;
; SM70-NEXT:    xor.b32 %r4, %r3, -1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r5;
; SM70-NEXT:    mov.b32 %r5, %r1;
; SM70-NEXT:    @%p1 bra $L__BB69_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @nand_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: nand_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:  $L__BB70_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r8;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r3, %r8, %r2;
; SM70-NEXT:    and.b32 %r4, %r7, %r1;
; SM70-NEXT:    xor.b32 %r5, %r4, -1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    xor.b32 %r6, %r3, -1;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB70_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @nand_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: nand_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r13, %rd2;
; SM70-NEXT:  $L__BB71_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r13;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r14;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r5, %r14, %r2;
; SM70-NEXT:    and.b32 %r6, %r13, %r1;
; SM70-NEXT:    xor.b32 %r7, %r6, -1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r7;
; SM70-NEXT:    xor.b32 %r8, %r5, -1;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r8;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r13, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB71_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r15, %rd12;
; SM70-NEXT:  $L__BB71_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd13, %r15;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r16;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    and.b32 %r9, %r16, %r4;
; SM70-NEXT:    and.b32 %r10, %r15, %r3;
; SM70-NEXT:    xor.b32 %r11, %r10, -1;
; SM70-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NEXT:    xor.b32 %r12, %r9, -1;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
; SM70-NEXT:    setp.ne.b64 %p2, %rd21, %rd16;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r15, %rd21;
; SM70-NEXT:    @%p2 bra $L__BB71_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @nand_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: nand_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<6>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [nand_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT:  $L__BB72_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r3, %r5, %r1;
; SM70-NEXT:    not.b32 %r4, %r3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM70-NEXT:    setp.ne.b32 %p1, %r2, %r5;
; SM70-NEXT:    mov.b32 %r5, %r2;
; SM70-NEXT:    @%p1 bra $L__BB72_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @nand_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: nand_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<13>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [nand_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:  $L__BB73_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r3, %r6, %r2;
; SM70-NEXT:    and.b32 %r4, %r5, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r4;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r3;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    not.b64 %rd11, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd12;
; SM70-NEXT:    @%p1 bra $L__BB73_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @nand_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: nand_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<24>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:  $L__BB74_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r5, %r10, %r2;
; SM70-NEXT:    and.b32 %r6, %r9, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r6;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r5;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    not.b64 %rd11, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd12;
; SM70-NEXT:    @%p1 bra $L__BB74_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd13;
; SM70-NEXT:  $L__BB74_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd14, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd15, %r12;
; SM70-NEXT:    shl.b64 %rd16, %rd15, 32;
; SM70-NEXT:    or.b64 %rd17, %rd14, %rd16;
; SM70-NEXT:    and.b32 %r7, %r12, %r4;
; SM70-NEXT:    and.b32 %r8, %r11, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r8;
; SM70-NEXT:    cvt.u64.u32 %rd19, %r7;
; SM70-NEXT:    shl.b64 %rd20, %rd19, 32;
; SM70-NEXT:    or.b64 %rd21, %rd18, %rd20;
; SM70-NEXT:    not.b64 %rd22, %rd21;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
; SM70-NEXT:    setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd23;
; SM70-NEXT:    @%p2 bra $L__BB74_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @nand_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: nand_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<5>;
; SM70-NEXT:    .reg .b32 %r<25>;
; SM70-NEXT:    .reg .b64 %rd<46>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NEXT:  $L__BB75_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r17;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r18;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r9, %r18, %r6;
; SM70-NEXT:    and.b32 %r10, %r17, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r10;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r9;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    not.b64 %rd11, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r17, %rd12;
; SM70-NEXT:    @%p1 bra $L__BB75_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM70-NEXT:    cvt.u32.u64 %r19, %rd13;
; SM70-NEXT:  $L__BB75_3: // %atomicrmw.start6
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd14, %r19;
; SM70-NEXT:    cvt.u64.u32 %rd15, %r20;
; SM70-NEXT:    shl.b64 %rd16, %rd15, 32;
; SM70-NEXT:    or.b64 %rd17, %rd14, %rd16;
; SM70-NEXT:    and.b32 %r11, %r20, %r8;
; SM70-NEXT:    and.b32 %r12, %r19, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NEXT:    cvt.u64.u32 %rd19, %r11;
; SM70-NEXT:    shl.b64 %rd20, %rd19, 32;
; SM70-NEXT:    or.b64 %rd21, %rd18, %rd20;
; SM70-NEXT:    not.b64 %rd22, %rd21;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
; SM70-NEXT:    setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM70-NEXT:    cvt.u32.u64 %r19, %rd23;
; SM70-NEXT:    @%p2 bra $L__BB75_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM70-NEXT:    cvt.u32.u64 %r21, %rd24;
; SM70-NEXT:  $L__BB75_5: // %atomicrmw.start16
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd25, %r21;
; SM70-NEXT:    cvt.u64.u32 %rd26, %r22;
; SM70-NEXT:    shl.b64 %rd27, %rd26, 32;
; SM70-NEXT:    or.b64 %rd28, %rd25, %rd27;
; SM70-NEXT:    and.b32 %r13, %r22, %r2;
; SM70-NEXT:    and.b32 %r14, %r21, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd29, %r14;
; SM70-NEXT:    cvt.u64.u32 %rd30, %r13;
; SM70-NEXT:    shl.b64 %rd31, %rd30, 32;
; SM70-NEXT:    or.b64 %rd32, %rd29, %rd31;
; SM70-NEXT:    not.b64 %rd33, %rd32;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
; SM70-NEXT:    setp.ne.b64 %p3, %rd34, %rd28;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM70-NEXT:    cvt.u32.u64 %r21, %rd34;
; SM70-NEXT:    @%p3 bra $L__BB75_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end15
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM70-NEXT:    cvt.u32.u64 %r23, %rd35;
; SM70-NEXT:  $L__BB75_7: // %atomicrmw.start22
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd36, %r23;
; SM70-NEXT:    cvt.u64.u32 %rd37, %r24;
; SM70-NEXT:    shl.b64 %rd38, %rd37, 32;
; SM70-NEXT:    or.b64 %rd39, %rd36, %rd38;
; SM70-NEXT:    and.b32 %r15, %r24, %r4;
; SM70-NEXT:    and.b32 %r16, %r23, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd40, %r16;
; SM70-NEXT:    cvt.u64.u32 %rd41, %r15;
; SM70-NEXT:    shl.b64 %rd42, %rd41, 32;
; SM70-NEXT:    or.b64 %rd43, %rd40, %rd42;
; SM70-NEXT:    not.b64 %rd44, %rd43;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
; SM70-NEXT:    setp.ne.b64 %p4, %rd45, %rd39;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM70-NEXT:    cvt.u32.u64 %r23, %rd45;
; SM70-NEXT:    @%p4 bra $L__BB75_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end21
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @nand_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: nand_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b64 %rd<7>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd3, [nand_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd6, [%rd3];
; SM70-NEXT:  $L__BB76_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b64 %rd4, %rd6, %rd1;
; SM70-NEXT:    not.b64 %rd5, %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
; SM70-NEXT:    setp.ne.b64 %p1, %rd2, %rd6;
; SM70-NEXT:    mov.b64 %rd6, %rd2;
; SM70-NEXT:    @%p1 bra $L__BB76_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @nand_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: nand_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd5, [nand_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd1}, [nand_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd10, [%rd5];
; SM70-NEXT:  $L__BB77_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd3, %rd10;
; SM70-NEXT:    and.b64 %rd6, %rd3, %rd2;
; SM70-NEXT:    not.b64 %rd7, %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd10, [%rd5], %rd3, %rd7;
; SM70-NEXT:    setp.ne.b64 %p1, %rd10, %rd3;
; SM70-NEXT:    @%p1 bra $L__BB77_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
; SM70-NEXT:  $L__BB77_3: // %atomicrmw.start4
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b64 %rd8, %rd11, %rd1;
; SM70-NEXT:    not.b64 %rd9, %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM70-NEXT:    setp.ne.b64 %p2, %rd4, %rd11;
; SM70-NEXT:    mov.b64 %rd11, %rd4;
; SM70-NEXT:    @%p2 bra $L__BB77_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end3
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @nand_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: nand_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<5>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd9, [nand_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd6}, [nand_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd3, %rd2}, [nand_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd18, [%rd9];
; SM70-NEXT:  $L__BB78_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd4, %rd18;
; SM70-NEXT:    and.b64 %rd10, %rd4, %rd3;
; SM70-NEXT:    not.b64 %rd11, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd4, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd18, %rd4;
; SM70-NEXT:    @%p1 bra $L__BB78_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
; SM70-NEXT:  $L__BB78_3: // %atomicrmw.start8
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd19;
; SM70-NEXT:    and.b64 %rd12, %rd5, %rd2;
; SM70-NEXT:    not.b64 %rd13, %rd12;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd5, %rd13;
; SM70-NEXT:    setp.ne.b64 %p2, %rd19, %rd5;
; SM70-NEXT:    @%p2 bra $L__BB78_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end7
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
; SM70-NEXT:  $L__BB78_5: // %atomicrmw.start20
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd7, %rd20;
; SM70-NEXT:    and.b64 %rd14, %rd7, %rd1;
; SM70-NEXT:    not.b64 %rd15, %rd14;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd20, [%rd9+16], %rd7, %rd15;
; SM70-NEXT:    setp.ne.b64 %p3, %rd20, %rd7;
; SM70-NEXT:    @%p3 bra $L__BB78_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end19
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
; SM70-NEXT:  $L__BB78_7: // %atomicrmw.start28
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b64 %rd16, %rd21, %rd6;
; SM70-NEXT:    not.b64 %rd17, %rd16;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM70-NEXT:    setp.ne.b64 %p4, %rd8, %rd21;
; SM70-NEXT:    mov.b64 %rd21, %rd8;
; SM70-NEXT:    @%p4 bra $L__BB78_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end27
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd20, %rd8};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @nand_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: nand_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<9>;
; SM70-NEXT:    .reg .b64 %rd<42>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd17, [nand_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd10, %rd14}, [nand_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd11}, [nand_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd7}, [nand_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd3}, [nand_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd34, [%rd17];
; SM70-NEXT:  $L__BB79_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd34;
; SM70-NEXT:    and.b64 %rd18, %rd5, %rd4;
; SM70-NEXT:    not.b64 %rd19, %rd18;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd5, %rd19;
; SM70-NEXT:    setp.ne.b64 %p1, %rd34, %rd5;
; SM70-NEXT:    @%p1 bra $L__BB79_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
; SM70-NEXT:  $L__BB79_3: // %atomicrmw.start12
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd6, %rd35;
; SM70-NEXT:    and.b64 %rd20, %rd6, %rd3;
; SM70-NEXT:    not.b64 %rd21, %rd20;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd6, %rd21;
; SM70-NEXT:    setp.ne.b64 %p2, %rd35, %rd6;
; SM70-NEXT:    @%p2 bra $L__BB79_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end11
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
; SM70-NEXT:  $L__BB79_5: // %atomicrmw.start24
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd8, %rd36;
; SM70-NEXT:    and.b64 %rd22, %rd8, %rd2;
; SM70-NEXT:    not.b64 %rd23, %rd22;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd8, %rd23;
; SM70-NEXT:    setp.ne.b64 %p3, %rd36, %rd8;
; SM70-NEXT:    @%p3 bra $L__BB79_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end23
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
; SM70-NEXT:  $L__BB79_7: // %atomicrmw.start32
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd9, %rd37;
; SM70-NEXT:    and.b64 %rd24, %rd9, %rd7;
; SM70-NEXT:    not.b64 %rd25, %rd24;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd9, %rd25;
; SM70-NEXT:    setp.ne.b64 %p4, %rd37, %rd9;
; SM70-NEXT:    @%p4 bra $L__BB79_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end31
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
; SM70-NEXT:  $L__BB79_9: // %atomicrmw.start48
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd12, %rd38;
; SM70-NEXT:    and.b64 %rd26, %rd12, %rd1;
; SM70-NEXT:    not.b64 %rd27, %rd26;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd12, %rd27;
; SM70-NEXT:    setp.ne.b64 %p5, %rd38, %rd12;
; SM70-NEXT:    @%p5 bra $L__BB79_9;
; SM70-NEXT:  // %bb.10: // %atomicrmw.end47
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
; SM70-NEXT:  $L__BB79_11: // %atomicrmw.start56
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd13, %rd39;
; SM70-NEXT:    and.b64 %rd28, %rd13, %rd11;
; SM70-NEXT:    not.b64 %rd29, %rd28;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd13, %rd29;
; SM70-NEXT:    setp.ne.b64 %p6, %rd39, %rd13;
; SM70-NEXT:    @%p6 bra $L__BB79_11;
; SM70-NEXT:  // %bb.12: // %atomicrmw.end55
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
; SM70-NEXT:  $L__BB79_13: // %atomicrmw.start68
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd15, %rd40;
; SM70-NEXT:    and.b64 %rd30, %rd15, %rd10;
; SM70-NEXT:    not.b64 %rd31, %rd30;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd40, [%rd17+48], %rd15, %rd31;
; SM70-NEXT:    setp.ne.b64 %p7, %rd40, %rd15;
; SM70-NEXT:    @%p7 bra $L__BB79_13;
; SM70-NEXT:  // %bb.14: // %atomicrmw.end67
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
; SM70-NEXT:  $L__BB79_15: // %atomicrmw.start76
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b64 %rd32, %rd41, %rd14;
; SM70-NEXT:    not.b64 %rd33, %rd32;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM70-NEXT:    setp.ne.b64 %p8, %rd16, %rd41;
; SM70-NEXT:    mov.b64 %rd41, %rd16;
; SM70-NEXT:    @%p8 bra $L__BB79_15;
; SM70-NEXT:  // %bb.16: // %atomicrmw.end75
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd40, %rd16};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd34, %rd35};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @or_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: or_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<8>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b8 %r1, [or_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd2, %rd1, -4;
; SM70-NEXT:    cvt.u32.u64 %r2, %rd1;
; SM70-NEXT:    and.b32 %r3, %r2, 3;
; SM70-NEXT:    shl.b32 %r4, %r3, 3;
; SM70-NEXT:    shl.b32 %r5, %r1, %r4;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
; SM70-NEXT:    shr.u32 %r7, %r6, %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r7;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @or_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: or_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<8>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd2, %rd1, -4;
; SM70-NEXT:    cvt.u32.u64 %r1, %rd1;
; SM70-NEXT:    and.b32 %r2, %r1, 3;
; SM70-NEXT:    shl.b32 %r3, %r2, 3;
; SM70-NEXT:    ld.param.b16 %r4, [or_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    shl.b32 %r5, %r4, %r3;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
; SM70-NEXT:    shr.u32 %r7, %r6, %r3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r7;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @or_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: or_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [or_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:  $L__BB82_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    or.b32 %r3, %r4, %r2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p1 bra $L__BB82_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @or_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: or_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:  $L__BB83_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    or.b32 %r3, %r5, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NEXT:    or.b32 %r4, %r6, %r2;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB83_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @or_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: or_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<8>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %r1, [or_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd2, %rd1, -4;
; SM70-NEXT:    cvt.u32.u64 %r2, %rd1;
; SM70-NEXT:    and.b32 %r3, %r2, 3;
; SM70-NEXT:    shl.b32 %r4, %r3, 3;
; SM70-NEXT:    shl.b32 %r5, %r1, %r4;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5;
; SM70-NEXT:    shr.u32 %r7, %r6, %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r7;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @or_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: or_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [or_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:  $L__BB85_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    or.b32 %r3, %r4, %r2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p1 bra $L__BB85_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @or_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: or_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:  $L__BB86_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    or.b32 %r3, %r5, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NEXT:    or.b32 %r4, %r6, %r2;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB86_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @or_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: or_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:  $L__BB87_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    or.b32 %r5, %r9, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    or.b32 %r6, %r10, %r2;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB87_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NEXT:  $L__BB87_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    or.b32 %r7, %r11, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NEXT:    or.b32 %r8, %r12, %r4;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NEXT:    @%p2 bra $L__BB87_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @or_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: or_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [or_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r2, [%rd1], %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @or_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: or_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [or_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r3, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r4, [%rd1+4], %r2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @or_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: or_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r5, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r6, [%rd1+4], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r7, [%rd1+8], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r8, [%rd1+12], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @or_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: or_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [or_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [or_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r9, [%rd1], %r5;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r10, [%rd1+4], %r6;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r11, [%rd1+8], %r7;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r12, [%rd1+12], %r8;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r13, [%rd1+16], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r14, [%rd1+20], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r15, [%rd1+24], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.or.b32 %r16, [%rd1+28], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @or_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: or_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<4>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd2, [or_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd3, [%rd1], %rd2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @or_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: or_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd4, [%rd1], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd5, [%rd1+8], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd4, %rd5};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @or_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: or_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<10>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd6, [%rd1], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd7, [%rd1+8], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd8, [%rd1+16], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd9, [%rd1+24], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @or_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: or_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<18>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [or_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [or_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [or_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd6, %rd7}, [or_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [or_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd10, [%rd1], %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd11, [%rd1+8], %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd12, [%rd1+16], %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd13, [%rd1+24], %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd14, [%rd1+32], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd15, [%rd1+40], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd16, [%rd1+48], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.or.b64 %rd17, [%rd1+56], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise or ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @xor_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: xor_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<8>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b8 %r1, [xor_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd2, %rd1, -4;
; SM70-NEXT:    cvt.u32.u64 %r2, %rd1;
; SM70-NEXT:    and.b32 %r3, %r2, 3;
; SM70-NEXT:    shl.b32 %r4, %r3, 3;
; SM70-NEXT:    shl.b32 %r5, %r1, %r4;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
; SM70-NEXT:    shr.u32 %r7, %r6, %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r7;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @xor_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: xor_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<8>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd2, %rd1, -4;
; SM70-NEXT:    cvt.u32.u64 %r1, %rd1;
; SM70-NEXT:    and.b32 %r2, %r1, 3;
; SM70-NEXT:    shl.b32 %r3, %r2, 3;
; SM70-NEXT:    ld.param.b16 %r4, [xor_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    shl.b32 %r5, %r4, %r3;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
; SM70-NEXT:    shr.u32 %r7, %r6, %r3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r7;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @xor_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: xor_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [xor_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:  $L__BB98_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    xor.b32 %r3, %r4, %r2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p1 bra $L__BB98_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @xor_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: xor_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:  $L__BB99_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    xor.b32 %r3, %r5, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NEXT:    xor.b32 %r4, %r6, %r2;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB99_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @xor_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: xor_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<8>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %r1, [xor_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd2, %rd1, -4;
; SM70-NEXT:    cvt.u32.u64 %r2, %rd1;
; SM70-NEXT:    and.b32 %r3, %r2, 3;
; SM70-NEXT:    shl.b32 %r4, %r3, 3;
; SM70-NEXT:    shl.b32 %r5, %r1, %r4;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5;
; SM70-NEXT:    shr.u32 %r7, %r6, %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r7;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @xor_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: xor_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [xor_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:  $L__BB101_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    xor.b32 %r3, %r4, %r2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p1 bra $L__BB101_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @xor_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: xor_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:  $L__BB102_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    xor.b32 %r3, %r5, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NEXT:    xor.b32 %r4, %r6, %r2;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB102_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @xor_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: xor_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:  $L__BB103_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    xor.b32 %r5, %r9, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    xor.b32 %r6, %r10, %r2;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB103_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NEXT:  $L__BB103_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    xor.b32 %r7, %r11, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NEXT:    xor.b32 %r8, %r12, %r4;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NEXT:    @%p2 bra $L__BB103_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @xor_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: xor_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [xor_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r2, [%rd1], %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @xor_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: xor_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [xor_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r3, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r4, [%rd1+4], %r2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @xor_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: xor_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r5, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r6, [%rd1+4], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r7, [%rd1+8], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r8, [%rd1+12], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @xor_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: xor_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [xor_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [xor_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r9, [%rd1], %r5;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r10, [%rd1+4], %r6;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r11, [%rd1+8], %r7;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r12, [%rd1+12], %r8;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r13, [%rd1+16], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r14, [%rd1+20], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r15, [%rd1+24], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b32 %r16, [%rd1+28], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @xor_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: xor_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<4>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd2, [xor_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd3, [%rd1], %rd2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @xor_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: xor_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd4, [%rd1], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd5, [%rd1+8], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd4, %rd5};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @xor_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: xor_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<10>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd6, [%rd1], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd7, [%rd1+8], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd8, [%rd1+16], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd9, [%rd1+24], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @xor_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: xor_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<18>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [xor_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [xor_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [xor_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd6, %rd7}, [xor_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [xor_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd10, [%rd1], %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd11, [%rd1+8], %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd12, [%rd1+16], %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd13, [%rd1+24], %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd14, [%rd1+32], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd15, [%rd1+40], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd16, [%rd1+48], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.xor.b64 %rd17, [%rd1+56], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise xor ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @max_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: max_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<4>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [max_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.s8 %rs1, [max_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 255, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:  $L__BB112_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r14, %r1;
; SM70-NEXT:    cvt.s8.s32 %rs2, %r7;
; SM70-NEXT:    max.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs3;
; SM70-NEXT:    and.b32 %r9, %r8, 255;
; SM70-NEXT:    shl.b32 %r10, %r9, %r1;
; SM70-NEXT:    and.b32 %r11, %r14, %r2;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r14;
; SM70-NEXT:    mov.b32 %r14, %r3;
; SM70-NEXT:    @%p1 bra $L__BB112_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @max_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: max_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<14>;
; SM70-NEXT:    .reg .b32 %r<16>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [max_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT:    ld.param.b64 %rd2, [max_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    and.b32 %r6, %r5, 3;
; SM70-NEXT:    shl.b32 %r1, %r6, 3;
; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r7;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
; SM70-NEXT:    cvt.s16.s8 %rs7, %rs5;
; SM70-NEXT:    cvt.s16.s8 %rs8, %rs6;
; SM70-NEXT:  $L__BB113_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r8, %r15, %r1;
; SM70-NEXT:    cvt.s8.s32 %rs3, %r8;
; SM70-NEXT:    bfe.s32 %r9, %r8, 8, 8;
; SM70-NEXT:    cvt.s8.s32 %rs4, %r9;
; SM70-NEXT:    max.s16 %rs9, %rs4, %rs8;
; SM70-NEXT:    max.s16 %rs10, %rs3, %rs7;
; SM70-NEXT:    and.b16 %rs11, %rs10, 255;
; SM70-NEXT:    shl.b16 %rs12, %rs9, 8;
; SM70-NEXT:    or.b16 %rs13, %rs11, %rs12;
; SM70-NEXT:    cvt.u32.u16 %r10, %rs13;
; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
; SM70-NEXT:    and.b32 %r12, %r15, %r2;
; SM70-NEXT:    or.b32 %r13, %r12, %r11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r15;
; SM70-NEXT:    mov.b32 %r15, %r3;
; SM70-NEXT:    @%p1 bra $L__BB113_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r14, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r14;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @max_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: max_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<6>;
; SM70-NEXT:    .reg .b32 %r<27>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [max_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [max_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x8880U;
; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
; SM70-NEXT:  $L__BB114_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r4, %r26, 0, 0x8880U;
; SM70-NEXT:    setp.gt.s32 %p1, %r4, %r3;
; SM70-NEXT:    prmt.b32 %r6, %r26, 0, 0x9991U;
; SM70-NEXT:    setp.gt.s32 %p2, %r6, %r5;
; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM70-NEXT:    prmt.b32 %r8, %r26, 0, 0xaaa2U;
; SM70-NEXT:    setp.gt.s32 %p3, %r8, %r7;
; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM70-NEXT:    prmt.b32 %r10, %r26, 0, 0xbbb3U;
; SM70-NEXT:    setp.gt.s32 %p4, %r10, %r9;
; SM70-NEXT:    prmt.b32 %r11, %r2, 0, 0x7770U;
; SM70-NEXT:    prmt.b32 %r12, %r2, 0, 0x7771U;
; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7772U;
; SM70-NEXT:    prmt.b32 %r14, %r2, 0, 0x7773U;
; SM70-NEXT:    prmt.b32 %r15, %r26, 0, 0x7773U;
; SM70-NEXT:    selp.b32 %r16, %r15, %r14, %p4;
; SM70-NEXT:    prmt.b32 %r17, %r26, 0, 0x7772U;
; SM70-NEXT:    selp.b32 %r18, %r17, %r13, %p3;
; SM70-NEXT:    prmt.b32 %r19, %r18, %r16, 0x3340U;
; SM70-NEXT:    prmt.b32 %r20, %r26, 0, 0x7771U;
; SM70-NEXT:    selp.b32 %r21, %r20, %r12, %p2;
; SM70-NEXT:    prmt.b32 %r22, %r26, 0, 0x7770U;
; SM70-NEXT:    selp.b32 %r23, %r22, %r11, %p1;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r21, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r24, %r19, 0x5410U;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NEXT:    setp.ne.b32 %p5, %r1, %r26;
; SM70-NEXT:    mov.b32 %r26, %r1;
; SM70-NEXT:    @%p5 bra $L__BB114_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @max_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: max_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<10>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<51>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [max_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-NEXT:  $L__BB115_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r49;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x8880U;
; SM70-NEXT:    prmt.b32 %r4, %r50, 0, 0x8880U;
; SM70-NEXT:    setp.gt.s32 %p1, %r4, %r3;
; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
; SM70-NEXT:    prmt.b32 %r6, %r50, 0, 0x9991U;
; SM70-NEXT:    setp.gt.s32 %p2, %r6, %r5;
; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM70-NEXT:    prmt.b32 %r8, %r50, 0, 0xaaa2U;
; SM70-NEXT:    setp.gt.s32 %p3, %r8, %r7;
; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM70-NEXT:    prmt.b32 %r10, %r50, 0, 0xbbb3U;
; SM70-NEXT:    setp.gt.s32 %p4, %r10, %r9;
; SM70-NEXT:    prmt.b32 %r11, %r1, 0, 0x8880U;
; SM70-NEXT:    prmt.b32 %r12, %r49, 0, 0x8880U;
; SM70-NEXT:    setp.gt.s32 %p5, %r12, %r11;
; SM70-NEXT:    prmt.b32 %r13, %r1, 0, 0x9991U;
; SM70-NEXT:    prmt.b32 %r14, %r49, 0, 0x9991U;
; SM70-NEXT:    setp.gt.s32 %p6, %r14, %r13;
; SM70-NEXT:    prmt.b32 %r15, %r1, 0, 0xaaa2U;
; SM70-NEXT:    prmt.b32 %r16, %r49, 0, 0xaaa2U;
; SM70-NEXT:    setp.gt.s32 %p7, %r16, %r15;
; SM70-NEXT:    prmt.b32 %r17, %r1, 0, 0xbbb3U;
; SM70-NEXT:    prmt.b32 %r18, %r49, 0, 0xbbb3U;
; SM70-NEXT:    setp.gt.s32 %p8, %r18, %r17;
; SM70-NEXT:    prmt.b32 %r19, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r19;
; SM70-NEXT:    prmt.b32 %r20, %r50, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r20;
; SM70-NEXT:    prmt.b32 %r21, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r21;
; SM70-NEXT:    prmt.b32 %r22, %r50, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r22;
; SM70-NEXT:    prmt.b32 %r23, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r23;
; SM70-NEXT:    prmt.b32 %r24, %r50, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs6, %r24;
; SM70-NEXT:    prmt.b32 %r25, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r25;
; SM70-NEXT:    prmt.b32 %r26, %r50, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r26;
; SM70-NEXT:    prmt.b32 %r27, %r1, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs9, %r27;
; SM70-NEXT:    prmt.b32 %r28, %r49, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r28;
; SM70-NEXT:    prmt.b32 %r29, %r1, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r29;
; SM70-NEXT:    prmt.b32 %r30, %r49, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs12, %r30;
; SM70-NEXT:    prmt.b32 %r31, %r1, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r31;
; SM70-NEXT:    prmt.b32 %r32, %r49, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs14, %r32;
; SM70-NEXT:    prmt.b32 %r33, %r1, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs15, %r33;
; SM70-NEXT:    prmt.b32 %r34, %r49, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs16, %r34;
; SM70-NEXT:    selp.b16 %rs17, %rs16, %rs15, %p8;
; SM70-NEXT:    cvt.u32.u16 %r35, %rs17;
; SM70-NEXT:    selp.b16 %rs18, %rs14, %rs13, %p7;
; SM70-NEXT:    cvt.u32.u16 %r36, %rs18;
; SM70-NEXT:    prmt.b32 %r37, %r36, %r35, 0x3340U;
; SM70-NEXT:    selp.b16 %rs19, %rs12, %rs11, %p6;
; SM70-NEXT:    cvt.u32.u16 %r38, %rs19;
; SM70-NEXT:    selp.b16 %rs20, %rs10, %rs9, %p5;
; SM70-NEXT:    cvt.u32.u16 %r39, %rs20;
; SM70-NEXT:    prmt.b32 %r40, %r39, %r38, 0x3340U;
; SM70-NEXT:    prmt.b32 %r41, %r40, %r37, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-NEXT:    selp.b16 %rs21, %rs8, %rs7, %p4;
; SM70-NEXT:    cvt.u32.u16 %r42, %rs21;
; SM70-NEXT:    selp.b16 %rs22, %rs6, %rs5, %p3;
; SM70-NEXT:    cvt.u32.u16 %r43, %rs22;
; SM70-NEXT:    prmt.b32 %r44, %r43, %r42, 0x3340U;
; SM70-NEXT:    selp.b16 %rs23, %rs4, %rs3, %p2;
; SM70-NEXT:    cvt.u32.u16 %r45, %rs23;
; SM70-NEXT:    selp.b16 %rs24, %rs2, %rs1, %p1;
; SM70-NEXT:    cvt.u32.u16 %r46, %rs24;
; SM70-NEXT:    prmt.b32 %r47, %r46, %r45, 0x3340U;
; SM70-NEXT:    prmt.b32 %r48, %r47, %r44, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r48;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-NEXT:    @%p9 bra $L__BB115_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @max_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: max_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<4>;
; SM70-NEXT:    .reg .b32 %r<14>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [max_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %rs1, [max_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NEXT:  $L__BB116_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    max.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs3;
; SM70-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r13;
; SM70-NEXT:    mov.b32 %r13, %r3;
; SM70-NEXT:    @%p1 bra $L__BB116_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @max_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: max_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<7>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [max_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [max_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT:  $L__BB117_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT:    max.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    max.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p1 bra $L__BB117_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @max_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: max_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [max_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT:  $L__BB118_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-NEXT:    max.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    max.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r3;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
; SM70-NEXT:    max.s16 %rs11, %rs10, %rs8;
; SM70-NEXT:    max.s16 %rs12, %rs9, %rs7;
; SM70-NEXT:    mov.b32 %r4, {%rs12, %rs11};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r4;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB118_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @max_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: max_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [max_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT:  $L__BB119_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-NEXT:    max.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    max.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r5, {%rs6, %rs5};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
; SM70-NEXT:    max.s16 %rs11, %rs10, %rs8;
; SM70-NEXT:    max.s16 %rs12, %rs9, %rs7;
; SM70-NEXT:    mov.b32 %r6, {%rs12, %rs11};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB119_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-NEXT:  $L__BB119_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    mov.b32 {%rs15, %rs16}, %r11;
; SM70-NEXT:    max.s16 %rs17, %rs16, %rs14;
; SM70-NEXT:    max.s16 %rs18, %rs15, %rs13;
; SM70-NEXT:    mov.b32 %r7, {%rs18, %rs17};
; SM70-NEXT:    cvt.u64.u32 %rd17, %r7;
; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
; SM70-NEXT:    max.s16 %rs23, %rs22, %rs20;
; SM70-NEXT:    max.s16 %rs24, %rs21, %rs19;
; SM70-NEXT:    mov.b32 %r8, {%rs24, %rs23};
; SM70-NEXT:    cvt.u64.u32 %rd18, %r8;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
; SM70-NEXT:    setp.ne.b64 %p2, %rd21, %rd16;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NEXT:    @%p2 bra $L__BB119_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @max_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: max_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [max_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [max_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r2, [%rd1], %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @max_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: max_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [max_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [max_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r3, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r4, [%rd1+4], %r2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @max_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: max_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [max_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r5, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r6, [%rd1+4], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r7, [%rd1+8], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r8, [%rd1+12], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @max_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: max_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [max_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [max_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [max_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r9, [%rd1], %r5;
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r10, [%rd1+4], %r6;
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r11, [%rd1+8], %r7;
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r12, [%rd1+12], %r8;
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r13, [%rd1+16], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r14, [%rd1+20], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r15, [%rd1+24], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.max.s32 %r16, [%rd1+28], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @max_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: max_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<4>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [max_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd2, [max_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd3, [%rd1], %rd2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @max_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: max_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [max_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd4, [%rd1], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd5, [%rd1+8], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd4, %rd5};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @max_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: max_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<10>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [max_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd6, [%rd1], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd7, [%rd1+8], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd8, [%rd1+16], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd9, [%rd1+24], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @max_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: max_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<18>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [max_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [max_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [max_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd6, %rd7}, [max_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [max_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd10, [%rd1], %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd11, [%rd1+8], %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd12, [%rd1+16], %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd13, [%rd1+24], %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd14, [%rd1+32], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd15, [%rd1+40], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd16, [%rd1+48], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.max.s64 %rd17, [%rd1+56], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise max ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @min_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: min_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<4>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [min_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.s8 %rs1, [min_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 255, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:  $L__BB128_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r14, %r1;
; SM70-NEXT:    cvt.s8.s32 %rs2, %r7;
; SM70-NEXT:    min.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs3;
; SM70-NEXT:    and.b32 %r9, %r8, 255;
; SM70-NEXT:    shl.b32 %r10, %r9, %r1;
; SM70-NEXT:    and.b32 %r11, %r14, %r2;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r14;
; SM70-NEXT:    mov.b32 %r14, %r3;
; SM70-NEXT:    @%p1 bra $L__BB128_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @min_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: min_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<14>;
; SM70-NEXT:    .reg .b32 %r<16>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [min_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT:    ld.param.b64 %rd2, [min_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    and.b32 %r6, %r5, 3;
; SM70-NEXT:    shl.b32 %r1, %r6, 3;
; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r7;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
; SM70-NEXT:    cvt.s16.s8 %rs7, %rs5;
; SM70-NEXT:    cvt.s16.s8 %rs8, %rs6;
; SM70-NEXT:  $L__BB129_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r8, %r15, %r1;
; SM70-NEXT:    cvt.s8.s32 %rs3, %r8;
; SM70-NEXT:    bfe.s32 %r9, %r8, 8, 8;
; SM70-NEXT:    cvt.s8.s32 %rs4, %r9;
; SM70-NEXT:    min.s16 %rs9, %rs4, %rs8;
; SM70-NEXT:    min.s16 %rs10, %rs3, %rs7;
; SM70-NEXT:    and.b16 %rs11, %rs10, 255;
; SM70-NEXT:    shl.b16 %rs12, %rs9, 8;
; SM70-NEXT:    or.b16 %rs13, %rs11, %rs12;
; SM70-NEXT:    cvt.u32.u16 %r10, %rs13;
; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
; SM70-NEXT:    and.b32 %r12, %r15, %r2;
; SM70-NEXT:    or.b32 %r13, %r12, %r11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r15;
; SM70-NEXT:    mov.b32 %r15, %r3;
; SM70-NEXT:    @%p1 bra $L__BB129_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r14, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r14;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @min_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: min_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<6>;
; SM70-NEXT:    .reg .b32 %r<27>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [min_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [min_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x8880U;
; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
; SM70-NEXT:  $L__BB130_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r4, %r26, 0, 0x8880U;
; SM70-NEXT:    setp.le.s32 %p1, %r4, %r3;
; SM70-NEXT:    prmt.b32 %r6, %r26, 0, 0x9991U;
; SM70-NEXT:    setp.le.s32 %p2, %r6, %r5;
; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM70-NEXT:    prmt.b32 %r8, %r26, 0, 0xaaa2U;
; SM70-NEXT:    setp.le.s32 %p3, %r8, %r7;
; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM70-NEXT:    prmt.b32 %r10, %r26, 0, 0xbbb3U;
; SM70-NEXT:    setp.le.s32 %p4, %r10, %r9;
; SM70-NEXT:    prmt.b32 %r11, %r2, 0, 0x7770U;
; SM70-NEXT:    prmt.b32 %r12, %r2, 0, 0x7771U;
; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7772U;
; SM70-NEXT:    prmt.b32 %r14, %r2, 0, 0x7773U;
; SM70-NEXT:    prmt.b32 %r15, %r26, 0, 0x7773U;
; SM70-NEXT:    selp.b32 %r16, %r15, %r14, %p4;
; SM70-NEXT:    prmt.b32 %r17, %r26, 0, 0x7772U;
; SM70-NEXT:    selp.b32 %r18, %r17, %r13, %p3;
; SM70-NEXT:    prmt.b32 %r19, %r18, %r16, 0x3340U;
; SM70-NEXT:    prmt.b32 %r20, %r26, 0, 0x7771U;
; SM70-NEXT:    selp.b32 %r21, %r20, %r12, %p2;
; SM70-NEXT:    prmt.b32 %r22, %r26, 0, 0x7770U;
; SM70-NEXT:    selp.b32 %r23, %r22, %r11, %p1;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r21, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r24, %r19, 0x5410U;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NEXT:    setp.ne.b32 %p5, %r1, %r26;
; SM70-NEXT:    mov.b32 %r26, %r1;
; SM70-NEXT:    @%p5 bra $L__BB130_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @min_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: min_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<10>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<51>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [min_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-NEXT:  $L__BB131_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r49;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x8880U;
; SM70-NEXT:    prmt.b32 %r4, %r50, 0, 0x8880U;
; SM70-NEXT:    setp.le.s32 %p1, %r4, %r3;
; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x9991U;
; SM70-NEXT:    prmt.b32 %r6, %r50, 0, 0x9991U;
; SM70-NEXT:    setp.le.s32 %p2, %r6, %r5;
; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0xaaa2U;
; SM70-NEXT:    prmt.b32 %r8, %r50, 0, 0xaaa2U;
; SM70-NEXT:    setp.le.s32 %p3, %r8, %r7;
; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0xbbb3U;
; SM70-NEXT:    prmt.b32 %r10, %r50, 0, 0xbbb3U;
; SM70-NEXT:    setp.le.s32 %p4, %r10, %r9;
; SM70-NEXT:    prmt.b32 %r11, %r1, 0, 0x8880U;
; SM70-NEXT:    prmt.b32 %r12, %r49, 0, 0x8880U;
; SM70-NEXT:    setp.le.s32 %p5, %r12, %r11;
; SM70-NEXT:    prmt.b32 %r13, %r1, 0, 0x9991U;
; SM70-NEXT:    prmt.b32 %r14, %r49, 0, 0x9991U;
; SM70-NEXT:    setp.le.s32 %p6, %r14, %r13;
; SM70-NEXT:    prmt.b32 %r15, %r1, 0, 0xaaa2U;
; SM70-NEXT:    prmt.b32 %r16, %r49, 0, 0xaaa2U;
; SM70-NEXT:    setp.le.s32 %p7, %r16, %r15;
; SM70-NEXT:    prmt.b32 %r17, %r1, 0, 0xbbb3U;
; SM70-NEXT:    prmt.b32 %r18, %r49, 0, 0xbbb3U;
; SM70-NEXT:    setp.le.s32 %p8, %r18, %r17;
; SM70-NEXT:    prmt.b32 %r19, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r19;
; SM70-NEXT:    prmt.b32 %r20, %r50, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r20;
; SM70-NEXT:    prmt.b32 %r21, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r21;
; SM70-NEXT:    prmt.b32 %r22, %r50, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r22;
; SM70-NEXT:    prmt.b32 %r23, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r23;
; SM70-NEXT:    prmt.b32 %r24, %r50, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs6, %r24;
; SM70-NEXT:    prmt.b32 %r25, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r25;
; SM70-NEXT:    prmt.b32 %r26, %r50, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r26;
; SM70-NEXT:    prmt.b32 %r27, %r1, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs9, %r27;
; SM70-NEXT:    prmt.b32 %r28, %r49, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r28;
; SM70-NEXT:    prmt.b32 %r29, %r1, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r29;
; SM70-NEXT:    prmt.b32 %r30, %r49, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs12, %r30;
; SM70-NEXT:    prmt.b32 %r31, %r1, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r31;
; SM70-NEXT:    prmt.b32 %r32, %r49, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs14, %r32;
; SM70-NEXT:    prmt.b32 %r33, %r1, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs15, %r33;
; SM70-NEXT:    prmt.b32 %r34, %r49, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs16, %r34;
; SM70-NEXT:    selp.b16 %rs17, %rs16, %rs15, %p8;
; SM70-NEXT:    cvt.u32.u16 %r35, %rs17;
; SM70-NEXT:    selp.b16 %rs18, %rs14, %rs13, %p7;
; SM70-NEXT:    cvt.u32.u16 %r36, %rs18;
; SM70-NEXT:    prmt.b32 %r37, %r36, %r35, 0x3340U;
; SM70-NEXT:    selp.b16 %rs19, %rs12, %rs11, %p6;
; SM70-NEXT:    cvt.u32.u16 %r38, %rs19;
; SM70-NEXT:    selp.b16 %rs20, %rs10, %rs9, %p5;
; SM70-NEXT:    cvt.u32.u16 %r39, %rs20;
; SM70-NEXT:    prmt.b32 %r40, %r39, %r38, 0x3340U;
; SM70-NEXT:    prmt.b32 %r41, %r40, %r37, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-NEXT:    selp.b16 %rs21, %rs8, %rs7, %p4;
; SM70-NEXT:    cvt.u32.u16 %r42, %rs21;
; SM70-NEXT:    selp.b16 %rs22, %rs6, %rs5, %p3;
; SM70-NEXT:    cvt.u32.u16 %r43, %rs22;
; SM70-NEXT:    prmt.b32 %r44, %r43, %r42, 0x3340U;
; SM70-NEXT:    selp.b16 %rs23, %rs4, %rs3, %p2;
; SM70-NEXT:    cvt.u32.u16 %r45, %rs23;
; SM70-NEXT:    selp.b16 %rs24, %rs2, %rs1, %p1;
; SM70-NEXT:    cvt.u32.u16 %r46, %rs24;
; SM70-NEXT:    prmt.b32 %r47, %r46, %r45, 0x3340U;
; SM70-NEXT:    prmt.b32 %r48, %r47, %r44, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r48;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-NEXT:    @%p9 bra $L__BB131_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @min_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: min_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<4>;
; SM70-NEXT:    .reg .b32 %r<14>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [min_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %rs1, [min_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NEXT:  $L__BB132_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    min.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs3;
; SM70-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r13;
; SM70-NEXT:    mov.b32 %r13, %r3;
; SM70-NEXT:    @%p1 bra $L__BB132_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @min_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: min_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<7>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [min_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [min_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT:  $L__BB133_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT:    min.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    min.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p1 bra $L__BB133_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @min_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: min_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [min_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT:  $L__BB134_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-NEXT:    min.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    min.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r3;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
; SM70-NEXT:    min.s16 %rs11, %rs10, %rs8;
; SM70-NEXT:    min.s16 %rs12, %rs9, %rs7;
; SM70-NEXT:    mov.b32 %r4, {%rs12, %rs11};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r4;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB134_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @min_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: min_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [min_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT:  $L__BB135_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-NEXT:    min.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    min.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r5, {%rs6, %rs5};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
; SM70-NEXT:    min.s16 %rs11, %rs10, %rs8;
; SM70-NEXT:    min.s16 %rs12, %rs9, %rs7;
; SM70-NEXT:    mov.b32 %r6, {%rs12, %rs11};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB135_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-NEXT:  $L__BB135_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    mov.b32 {%rs15, %rs16}, %r11;
; SM70-NEXT:    min.s16 %rs17, %rs16, %rs14;
; SM70-NEXT:    min.s16 %rs18, %rs15, %rs13;
; SM70-NEXT:    mov.b32 %r7, {%rs18, %rs17};
; SM70-NEXT:    cvt.u64.u32 %rd17, %r7;
; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
; SM70-NEXT:    min.s16 %rs23, %rs22, %rs20;
; SM70-NEXT:    min.s16 %rs24, %rs21, %rs19;
; SM70-NEXT:    mov.b32 %r8, {%rs24, %rs23};
; SM70-NEXT:    cvt.u64.u32 %rd18, %r8;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
; SM70-NEXT:    setp.ne.b64 %p2, %rd21, %rd16;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NEXT:    @%p2 bra $L__BB135_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @min_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: min_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [min_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [min_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r2, [%rd1], %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @min_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: min_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [min_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [min_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r3, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r4, [%rd1+4], %r2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @min_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: min_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [min_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r5, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r6, [%rd1+4], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r7, [%rd1+8], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r8, [%rd1+12], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @min_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: min_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [min_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [min_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [min_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r9, [%rd1], %r5;
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r10, [%rd1+4], %r6;
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r11, [%rd1+8], %r7;
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r12, [%rd1+12], %r8;
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r13, [%rd1+16], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r14, [%rd1+20], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r15, [%rd1+24], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.min.s32 %r16, [%rd1+28], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @min_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: min_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<4>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [min_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd2, [min_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd3, [%rd1], %rd2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @min_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: min_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [min_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd4, [%rd1], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd5, [%rd1+8], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd4, %rd5};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @min_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: min_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<10>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [min_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd6, [%rd1], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd7, [%rd1+8], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd8, [%rd1+16], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd9, [%rd1+24], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @min_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: min_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<18>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [min_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [min_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [min_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd6, %rd7}, [min_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [min_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd10, [%rd1], %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd11, [%rd1+8], %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd12, [%rd1+16], %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd13, [%rd1+24], %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd14, [%rd1+32], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd15, [%rd1+40], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd16, [%rd1+48], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.min.s64 %rd17, [%rd1+56], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise min ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @umax_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: umax_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<5>;
; SM70-NEXT:    .reg .b32 %r<14>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b8 %rs1, [umax_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd2, [umax_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 255, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NEXT:  $L__BB144_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    and.b16 %rs3, %rs2, 255;
; SM70-NEXT:    max.u16 %rs4, %rs3, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs4;
; SM70-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r13;
; SM70-NEXT:    mov.b32 %r13, %r3;
; SM70-NEXT:    @%p1 bra $L__BB144_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r12;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @umax_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: umax_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [umax_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT:    ld.param.b64 %rd2, [umax_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    and.b32 %r6, %r5, 3;
; SM70-NEXT:    shl.b32 %r1, %r6, 3;
; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r7;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
; SM70-NEXT:  $L__BB145_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r8, %r16, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r8;
; SM70-NEXT:    shr.u16 %rs4, %rs3, 8;
; SM70-NEXT:    mov.b32 %r9, {%rs3, %rs4};
; SM70-NEXT:    and.b32 %r10, %r9, 16711935;
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r10;
; SM70-NEXT:    max.u16 %rs9, %rs5, %rs7;
; SM70-NEXT:    max.u16 %rs10, %rs6, %rs8;
; SM70-NEXT:    shl.b16 %rs11, %rs10, 8;
; SM70-NEXT:    or.b16 %rs12, %rs9, %rs11;
; SM70-NEXT:    cvt.u32.u16 %r11, %rs12;
; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r16;
; SM70-NEXT:    mov.b32 %r16, %r3;
; SM70-NEXT:    @%p1 bra $L__BB145_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @umax_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: umax_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<6>;
; SM70-NEXT:    .reg .b32 %r<19>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [umax_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [umax_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT:  $L__BB146_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7770U;
; SM70-NEXT:    setp.gt.u32 %p1, %r4, %r3;
; SM70-NEXT:    prmt.b32 %r6, %r18, 0, 0x7771U;
; SM70-NEXT:    setp.gt.u32 %p2, %r6, %r5;
; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
; SM70-NEXT:    prmt.b32 %r8, %r18, 0, 0x7772U;
; SM70-NEXT:    setp.gt.u32 %p3, %r8, %r7;
; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0x7773U;
; SM70-NEXT:    prmt.b32 %r10, %r18, 0, 0x7773U;
; SM70-NEXT:    setp.gt.u32 %p4, %r10, %r9;
; SM70-NEXT:    selp.b32 %r11, %r10, %r9, %p4;
; SM70-NEXT:    selp.b32 %r12, %r8, %r7, %p3;
; SM70-NEXT:    prmt.b32 %r13, %r12, %r11, 0x3340U;
; SM70-NEXT:    selp.b32 %r14, %r6, %r5, %p2;
; SM70-NEXT:    selp.b32 %r15, %r4, %r3, %p1;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r14, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r13, 0x5410U;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT:    setp.ne.b32 %p5, %r1, %r18;
; SM70-NEXT:    mov.b32 %r18, %r1;
; SM70-NEXT:    @%p5 bra $L__BB146_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @umax_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: umax_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<10>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<35>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [umax_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NEXT:  $L__BB147_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r33;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r34;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
; SM70-NEXT:    prmt.b32 %r4, %r34, 0, 0x7770U;
; SM70-NEXT:    setp.gt.u32 %p1, %r4, %r3;
; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT:    prmt.b32 %r6, %r34, 0, 0x7771U;
; SM70-NEXT:    setp.gt.u32 %p2, %r6, %r5;
; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
; SM70-NEXT:    prmt.b32 %r8, %r34, 0, 0x7772U;
; SM70-NEXT:    setp.gt.u32 %p3, %r8, %r7;
; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0x7773U;
; SM70-NEXT:    prmt.b32 %r10, %r34, 0, 0x7773U;
; SM70-NEXT:    setp.gt.u32 %p4, %r10, %r9;
; SM70-NEXT:    prmt.b32 %r11, %r1, 0, 0x7770U;
; SM70-NEXT:    prmt.b32 %r12, %r33, 0, 0x7770U;
; SM70-NEXT:    setp.gt.u32 %p5, %r12, %r11;
; SM70-NEXT:    prmt.b32 %r13, %r1, 0, 0x7771U;
; SM70-NEXT:    prmt.b32 %r14, %r33, 0, 0x7771U;
; SM70-NEXT:    setp.gt.u32 %p6, %r14, %r13;
; SM70-NEXT:    prmt.b32 %r15, %r1, 0, 0x7772U;
; SM70-NEXT:    prmt.b32 %r16, %r33, 0, 0x7772U;
; SM70-NEXT:    setp.gt.u32 %p7, %r16, %r15;
; SM70-NEXT:    prmt.b32 %r17, %r1, 0, 0x7773U;
; SM70-NEXT:    prmt.b32 %r18, %r33, 0, 0x7773U;
; SM70-NEXT:    setp.gt.u32 %p8, %r18, %r17;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r5;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
; SM70-NEXT:    cvt.u16.u32 %rs6, %r8;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r9;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r10;
; SM70-NEXT:    cvt.u16.u32 %rs9, %r11;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r12;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r13;
; SM70-NEXT:    cvt.u16.u32 %rs12, %r14;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r15;
; SM70-NEXT:    cvt.u16.u32 %rs14, %r16;
; SM70-NEXT:    cvt.u16.u32 %rs15, %r17;
; SM70-NEXT:    cvt.u16.u32 %rs16, %r18;
; SM70-NEXT:    selp.b16 %rs17, %rs16, %rs15, %p8;
; SM70-NEXT:    cvt.u32.u16 %r19, %rs17;
; SM70-NEXT:    selp.b16 %rs18, %rs14, %rs13, %p7;
; SM70-NEXT:    cvt.u32.u16 %r20, %rs18;
; SM70-NEXT:    prmt.b32 %r21, %r20, %r19, 0x3340U;
; SM70-NEXT:    selp.b16 %rs19, %rs12, %rs11, %p6;
; SM70-NEXT:    cvt.u32.u16 %r22, %rs19;
; SM70-NEXT:    selp.b16 %rs20, %rs10, %rs9, %p5;
; SM70-NEXT:    cvt.u32.u16 %r23, %rs20;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r22, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r24, %r21, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r25;
; SM70-NEXT:    selp.b16 %rs21, %rs8, %rs7, %p4;
; SM70-NEXT:    cvt.u32.u16 %r26, %rs21;
; SM70-NEXT:    selp.b16 %rs22, %rs6, %rs5, %p3;
; SM70-NEXT:    cvt.u32.u16 %r27, %rs22;
; SM70-NEXT:    prmt.b32 %r28, %r27, %r26, 0x3340U;
; SM70-NEXT:    selp.b16 %rs23, %rs4, %rs3, %p2;
; SM70-NEXT:    cvt.u32.u16 %r29, %rs23;
; SM70-NEXT:    selp.b16 %rs24, %rs2, %rs1, %p1;
; SM70-NEXT:    cvt.u32.u16 %r30, %rs24;
; SM70-NEXT:    prmt.b32 %r31, %r30, %r29, 0x3340U;
; SM70-NEXT:    prmt.b32 %r32, %r31, %r28, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r32;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NEXT:    @%p9 bra $L__BB147_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @umax_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: umax_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<4>;
; SM70-NEXT:    .reg .b32 %r<14>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [umax_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %rs1, [umax_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NEXT:  $L__BB148_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    max.u16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs3;
; SM70-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r13;
; SM70-NEXT:    mov.b32 %r13, %r3;
; SM70-NEXT:    @%p1 bra $L__BB148_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @umax_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: umax_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<7>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [umax_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [umax_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT:  $L__BB149_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT:    max.u16 %rs5, %rs4, %rs2;
; SM70-NEXT:    max.u16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p1 bra $L__BB149_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @umax_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: umax_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [umax_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT:  $L__BB150_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-NEXT:    max.u16 %rs5, %rs4, %rs2;
; SM70-NEXT:    max.u16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r3;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
; SM70-NEXT:    max.u16 %rs11, %rs10, %rs8;
; SM70-NEXT:    max.u16 %rs12, %rs9, %rs7;
; SM70-NEXT:    mov.b32 %r4, {%rs12, %rs11};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r4;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB150_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @umax_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: umax_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umax_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT:  $L__BB151_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-NEXT:    max.u16 %rs5, %rs4, %rs2;
; SM70-NEXT:    max.u16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r5, {%rs6, %rs5};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
; SM70-NEXT:    max.u16 %rs11, %rs10, %rs8;
; SM70-NEXT:    max.u16 %rs12, %rs9, %rs7;
; SM70-NEXT:    mov.b32 %r6, {%rs12, %rs11};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB151_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-NEXT:  $L__BB151_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    mov.b32 {%rs15, %rs16}, %r11;
; SM70-NEXT:    max.u16 %rs17, %rs16, %rs14;
; SM70-NEXT:    max.u16 %rs18, %rs15, %rs13;
; SM70-NEXT:    mov.b32 %r7, {%rs18, %rs17};
; SM70-NEXT:    cvt.u64.u32 %rd17, %r7;
; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
; SM70-NEXT:    max.u16 %rs23, %rs22, %rs20;
; SM70-NEXT:    max.u16 %rs24, %rs21, %rs19;
; SM70-NEXT:    mov.b32 %r8, {%rs24, %rs23};
; SM70-NEXT:    cvt.u64.u32 %rd18, %r8;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
; SM70-NEXT:    setp.ne.b64 %p2, %rd21, %rd16;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NEXT:    @%p2 bra $L__BB151_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @umax_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: umax_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umax_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [umax_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r2, [%rd1], %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @umax_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: umax_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umax_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [umax_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r3, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r4, [%rd1+4], %r2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @umax_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: umax_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umax_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r5, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r6, [%rd1+4], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r7, [%rd1+8], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r8, [%rd1+12], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @umax_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: umax_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umax_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umax_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umax_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r9, [%rd1], %r5;
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r10, [%rd1+4], %r6;
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r11, [%rd1+8], %r7;
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r12, [%rd1+12], %r8;
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r13, [%rd1+16], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r14, [%rd1+20], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r15, [%rd1+24], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.max.u32 %r16, [%rd1+28], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @umax_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: umax_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<4>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umax_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd2, [umax_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd3, [%rd1], %rd2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @umax_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: umax_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umax_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd4, [%rd1], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd5, [%rd1+8], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd4, %rd5};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @umax_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: umax_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<10>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umax_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd6, [%rd1], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd7, [%rd1+8], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd8, [%rd1+16], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd9, [%rd1+24], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @umax_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: umax_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<18>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umax_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [umax_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [umax_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd6, %rd7}, [umax_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [umax_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd10, [%rd1], %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd11, [%rd1+8], %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd12, [%rd1+16], %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd13, [%rd1+24], %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd14, [%rd1+32], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd15, [%rd1+40], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd16, [%rd1+48], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.max.u64 %rd17, [%rd1+56], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umax ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @umin_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: umin_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<5>;
; SM70-NEXT:    .reg .b32 %r<14>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b8 %rs1, [umin_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd2, [umin_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 255, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NEXT:  $L__BB160_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    and.b16 %rs3, %rs2, 255;
; SM70-NEXT:    min.u16 %rs4, %rs3, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs4;
; SM70-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r13;
; SM70-NEXT:    mov.b32 %r13, %r3;
; SM70-NEXT:    @%p1 bra $L__BB160_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r12;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @umin_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: umin_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [umin_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT:    ld.param.b64 %rd2, [umin_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    and.b32 %r6, %r5, 3;
; SM70-NEXT:    shl.b32 %r1, %r6, 3;
; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r7;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
; SM70-NEXT:  $L__BB161_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r8, %r16, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r8;
; SM70-NEXT:    shr.u16 %rs4, %rs3, 8;
; SM70-NEXT:    mov.b32 %r9, {%rs3, %rs4};
; SM70-NEXT:    and.b32 %r10, %r9, 16711935;
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r10;
; SM70-NEXT:    min.u16 %rs9, %rs5, %rs7;
; SM70-NEXT:    min.u16 %rs10, %rs6, %rs8;
; SM70-NEXT:    shl.b16 %rs11, %rs10, 8;
; SM70-NEXT:    or.b16 %rs12, %rs9, %rs11;
; SM70-NEXT:    cvt.u32.u16 %r11, %rs12;
; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r16;
; SM70-NEXT:    mov.b32 %r16, %r3;
; SM70-NEXT:    @%p1 bra $L__BB161_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @umin_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: umin_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<6>;
; SM70-NEXT:    .reg .b32 %r<19>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [umin_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [umin_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT:  $L__BB162_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7770U;
; SM70-NEXT:    setp.le.u32 %p1, %r4, %r3;
; SM70-NEXT:    prmt.b32 %r6, %r18, 0, 0x7771U;
; SM70-NEXT:    setp.le.u32 %p2, %r6, %r5;
; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
; SM70-NEXT:    prmt.b32 %r8, %r18, 0, 0x7772U;
; SM70-NEXT:    setp.le.u32 %p3, %r8, %r7;
; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0x7773U;
; SM70-NEXT:    prmt.b32 %r10, %r18, 0, 0x7773U;
; SM70-NEXT:    setp.le.u32 %p4, %r10, %r9;
; SM70-NEXT:    selp.b32 %r11, %r10, %r9, %p4;
; SM70-NEXT:    selp.b32 %r12, %r8, %r7, %p3;
; SM70-NEXT:    prmt.b32 %r13, %r12, %r11, 0x3340U;
; SM70-NEXT:    selp.b32 %r14, %r6, %r5, %p2;
; SM70-NEXT:    selp.b32 %r15, %r4, %r3, %p1;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r14, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r13, 0x5410U;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT:    setp.ne.b32 %p5, %r1, %r18;
; SM70-NEXT:    mov.b32 %r18, %r1;
; SM70-NEXT:    @%p5 bra $L__BB162_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @umin_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: umin_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<10>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<35>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [umin_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NEXT:  $L__BB163_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r33;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r34;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
; SM70-NEXT:    prmt.b32 %r4, %r34, 0, 0x7770U;
; SM70-NEXT:    setp.le.u32 %p1, %r4, %r3;
; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT:    prmt.b32 %r6, %r34, 0, 0x7771U;
; SM70-NEXT:    setp.le.u32 %p2, %r6, %r5;
; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
; SM70-NEXT:    prmt.b32 %r8, %r34, 0, 0x7772U;
; SM70-NEXT:    setp.le.u32 %p3, %r8, %r7;
; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0x7773U;
; SM70-NEXT:    prmt.b32 %r10, %r34, 0, 0x7773U;
; SM70-NEXT:    setp.le.u32 %p4, %r10, %r9;
; SM70-NEXT:    prmt.b32 %r11, %r1, 0, 0x7770U;
; SM70-NEXT:    prmt.b32 %r12, %r33, 0, 0x7770U;
; SM70-NEXT:    setp.le.u32 %p5, %r12, %r11;
; SM70-NEXT:    prmt.b32 %r13, %r1, 0, 0x7771U;
; SM70-NEXT:    prmt.b32 %r14, %r33, 0, 0x7771U;
; SM70-NEXT:    setp.le.u32 %p6, %r14, %r13;
; SM70-NEXT:    prmt.b32 %r15, %r1, 0, 0x7772U;
; SM70-NEXT:    prmt.b32 %r16, %r33, 0, 0x7772U;
; SM70-NEXT:    setp.le.u32 %p7, %r16, %r15;
; SM70-NEXT:    prmt.b32 %r17, %r1, 0, 0x7773U;
; SM70-NEXT:    prmt.b32 %r18, %r33, 0, 0x7773U;
; SM70-NEXT:    setp.le.u32 %p8, %r18, %r17;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r5;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
; SM70-NEXT:    cvt.u16.u32 %rs6, %r8;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r9;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r10;
; SM70-NEXT:    cvt.u16.u32 %rs9, %r11;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r12;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r13;
; SM70-NEXT:    cvt.u16.u32 %rs12, %r14;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r15;
; SM70-NEXT:    cvt.u16.u32 %rs14, %r16;
; SM70-NEXT:    cvt.u16.u32 %rs15, %r17;
; SM70-NEXT:    cvt.u16.u32 %rs16, %r18;
; SM70-NEXT:    selp.b16 %rs17, %rs16, %rs15, %p8;
; SM70-NEXT:    cvt.u32.u16 %r19, %rs17;
; SM70-NEXT:    selp.b16 %rs18, %rs14, %rs13, %p7;
; SM70-NEXT:    cvt.u32.u16 %r20, %rs18;
; SM70-NEXT:    prmt.b32 %r21, %r20, %r19, 0x3340U;
; SM70-NEXT:    selp.b16 %rs19, %rs12, %rs11, %p6;
; SM70-NEXT:    cvt.u32.u16 %r22, %rs19;
; SM70-NEXT:    selp.b16 %rs20, %rs10, %rs9, %p5;
; SM70-NEXT:    cvt.u32.u16 %r23, %rs20;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r22, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r24, %r21, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r25;
; SM70-NEXT:    selp.b16 %rs21, %rs8, %rs7, %p4;
; SM70-NEXT:    cvt.u32.u16 %r26, %rs21;
; SM70-NEXT:    selp.b16 %rs22, %rs6, %rs5, %p3;
; SM70-NEXT:    cvt.u32.u16 %r27, %rs22;
; SM70-NEXT:    prmt.b32 %r28, %r27, %r26, 0x3340U;
; SM70-NEXT:    selp.b16 %rs23, %rs4, %rs3, %p2;
; SM70-NEXT:    cvt.u32.u16 %r29, %rs23;
; SM70-NEXT:    selp.b16 %rs24, %rs2, %rs1, %p1;
; SM70-NEXT:    cvt.u32.u16 %r30, %rs24;
; SM70-NEXT:    prmt.b32 %r31, %r30, %r29, 0x3340U;
; SM70-NEXT:    prmt.b32 %r32, %r31, %r28, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r32;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NEXT:    @%p9 bra $L__BB163_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @umin_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: umin_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<4>;
; SM70-NEXT:    .reg .b32 %r<14>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [umin_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %rs1, [umin_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NEXT:  $L__BB164_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    min.u16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs3;
; SM70-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r13;
; SM70-NEXT:    mov.b32 %r13, %r3;
; SM70-NEXT:    @%p1 bra $L__BB164_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @umin_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: umin_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<7>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [umin_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [umin_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT:  $L__BB165_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT:    min.u16 %rs5, %rs4, %rs2;
; SM70-NEXT:    min.u16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p1 bra $L__BB165_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @umin_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: umin_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [umin_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT:  $L__BB166_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-NEXT:    min.u16 %rs5, %rs4, %rs2;
; SM70-NEXT:    min.u16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r3;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r6;
; SM70-NEXT:    min.u16 %rs11, %rs10, %rs8;
; SM70-NEXT:    min.u16 %rs12, %rs9, %rs7;
; SM70-NEXT:    mov.b32 %r4, {%rs12, %rs11};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r4;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB166_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @umin_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: umin_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umin_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NEXT:  $L__BB167_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-NEXT:    min.u16 %rs5, %rs4, %rs2;
; SM70-NEXT:    min.u16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r5, {%rs6, %rs5};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r10;
; SM70-NEXT:    min.u16 %rs11, %rs10, %rs8;
; SM70-NEXT:    min.u16 %rs12, %rs9, %rs7;
; SM70-NEXT:    mov.b32 %r6, {%rs12, %rs11};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB167_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-NEXT:  $L__BB167_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    mov.b32 {%rs15, %rs16}, %r11;
; SM70-NEXT:    min.u16 %rs17, %rs16, %rs14;
; SM70-NEXT:    min.u16 %rs18, %rs15, %rs13;
; SM70-NEXT:    mov.b32 %r7, {%rs18, %rs17};
; SM70-NEXT:    cvt.u64.u32 %rd17, %r7;
; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
; SM70-NEXT:    min.u16 %rs23, %rs22, %rs20;
; SM70-NEXT:    min.u16 %rs24, %rs21, %rs19;
; SM70-NEXT:    mov.b32 %r8, {%rs24, %rs23};
; SM70-NEXT:    cvt.u64.u32 %rd18, %r8;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
; SM70-NEXT:    setp.ne.b64 %p2, %rd21, %rd16;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NEXT:    @%p2 bra $L__BB167_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @umin_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: umin_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umin_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [umin_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r2, [%rd1], %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @umin_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: umin_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umin_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [umin_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r3, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r4, [%rd1+4], %r2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @umin_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: umin_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umin_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r5, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r6, [%rd1+4], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r7, [%rd1+8], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r8, [%rd1+12], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @umin_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: umin_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umin_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [umin_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [umin_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r9, [%rd1], %r5;
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r10, [%rd1+4], %r6;
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r11, [%rd1+8], %r7;
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r12, [%rd1+12], %r8;
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r13, [%rd1+16], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r14, [%rd1+20], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r15, [%rd1+24], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.min.u32 %r16, [%rd1+28], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @umin_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: umin_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<4>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umin_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd2, [umin_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd3, [%rd1], %rd2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @umin_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: umin_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umin_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd4, [%rd1], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd5, [%rd1+8], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd4, %rd5};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @umin_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: umin_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<10>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umin_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd6, [%rd1], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd7, [%rd1+8], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd8, [%rd1+16], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd9, [%rd1+24], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @umin_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: umin_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<18>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [umin_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [umin_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [umin_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd6, %rd7}, [umin_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [umin_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd10, [%rd1], %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd11, [%rd1+8], %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd12, [%rd1+16], %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd13, [%rd1+24], %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd14, [%rd1+32], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd15, [%rd1+40], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd16, [%rd1+48], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.min.u64 %rd17, [%rd1+56], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise umin ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @uinc_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b16 %rs<6>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b8 %rs1, [uinc_wrap_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 255, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:  $L__BB176_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r14, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    and.b16 %rs3, %rs2, 255;
; SM70-NEXT:    add.s16 %rs4, %rs2, 1;
; SM70-NEXT:    setp.ge.u16 %p1, %rs3, %rs1;
; SM70-NEXT:    selp.b16 %rs5, 0, %rs4, %p1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs5;
; SM70-NEXT:    and.b32 %r9, %r8, 255;
; SM70-NEXT:    shl.b32 %r10, %r9, %r1;
; SM70-NEXT:    and.b32 %r11, %r14, %r2;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p2, %r3, %r14;
; SM70-NEXT:    mov.b32 %r14, %r3;
; SM70-NEXT:    @%p2 bra $L__BB176_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @uinc_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<4>;
; SM70-NEXT:    .reg .b16 %rs<15>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [uinc_wrap_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT:    ld.param.b64 %rd2, [uinc_wrap_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    and.b32 %r6, %r5, 3;
; SM70-NEXT:    shl.b32 %r1, %r6, 3;
; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r7;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:    mov.b32 {%rs8, %rs9}, %r4;
; SM70-NEXT:  $L__BB177_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r8, %r14, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r8;
; SM70-NEXT:    and.b16 %rs4, %rs3, 255;
; SM70-NEXT:    shr.u16 %rs5, %rs3, 8;
; SM70-NEXT:    add.s16 %rs6, %rs5, 1;
; SM70-NEXT:    add.s16 %rs7, %rs3, 1;
; SM70-NEXT:    setp.ge.u16 %p1, %rs5, %rs9;
; SM70-NEXT:    setp.ge.u16 %p2, %rs4, %rs8;
; SM70-NEXT:    selp.b16 %rs10, 0, %rs7, %p2;
; SM70-NEXT:    selp.b16 %rs11, 0, %rs6, %p1;
; SM70-NEXT:    shl.b16 %rs12, %rs11, 8;
; SM70-NEXT:    and.b16 %rs13, %rs10, 255;
; SM70-NEXT:    or.b16 %rs14, %rs13, %rs12;
; SM70-NEXT:    cvt.u32.u16 %r9, %rs14;
; SM70-NEXT:    shl.b32 %r10, %r9, %r1;
; SM70-NEXT:    and.b32 %r11, %r14, %r2;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p3, %r3, %r14;
; SM70-NEXT:    mov.b32 %r14, %r3;
; SM70-NEXT:    @%p3 bra $L__BB177_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @uinc_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<6>;
; SM70-NEXT:    .reg .b16 %rs<9>;
; SM70-NEXT:    .reg .b32 %r<23>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [uinc_wrap_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r22, [%rd1];
; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
; SM70-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
; SM70-NEXT:  $L__BB178_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r3, %r22, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    add.s16 %rs2, %rs1, 1;
; SM70-NEXT:    prmt.b32 %r4, %r22, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r4;
; SM70-NEXT:    add.s16 %rs4, %rs3, 1;
; SM70-NEXT:    prmt.b32 %r5, %r22, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r5;
; SM70-NEXT:    add.s16 %rs6, %rs5, 1;
; SM70-NEXT:    prmt.b32 %r6, %r22, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r6;
; SM70-NEXT:    add.s16 %rs8, %rs7, 1;
; SM70-NEXT:    setp.ge.u32 %p1, %r3, %r7;
; SM70-NEXT:    setp.ge.u32 %p2, %r4, %r8;
; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0x7772U;
; SM70-NEXT:    setp.ge.u32 %p3, %r5, %r9;
; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7773U;
; SM70-NEXT:    setp.ge.u32 %p4, %r6, %r10;
; SM70-NEXT:    cvt.u32.u16 %r11, %rs8;
; SM70-NEXT:    selp.b32 %r12, 0, %r11, %p4;
; SM70-NEXT:    cvt.u32.u16 %r13, %rs6;
; SM70-NEXT:    selp.b32 %r14, 0, %r13, %p3;
; SM70-NEXT:    prmt.b32 %r15, %r14, %r12, 0x3340U;
; SM70-NEXT:    cvt.u32.u16 %r16, %rs4;
; SM70-NEXT:    selp.b32 %r17, 0, %r16, %p2;
; SM70-NEXT:    cvt.u32.u16 %r18, %rs2;
; SM70-NEXT:    selp.b32 %r19, 0, %r18, %p1;
; SM70-NEXT:    prmt.b32 %r20, %r19, %r17, 0x3340U;
; SM70-NEXT:    prmt.b32 %r21, %r20, %r15, 0x5410U;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
; SM70-NEXT:    setp.ne.b32 %p5, %r1, %r22;
; SM70-NEXT:    mov.b32 %r22, %r1;
; SM70-NEXT:    @%p5 bra $L__BB178_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @uinc_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<10>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<35>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NEXT:  $L__BB179_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r33;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r34;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    prmt.b32 %r3, %r34, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    add.s16 %rs2, %rs1, 1;
; SM70-NEXT:    prmt.b32 %r4, %r34, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r4;
; SM70-NEXT:    add.s16 %rs4, %rs3, 1;
; SM70-NEXT:    prmt.b32 %r5, %r34, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r5;
; SM70-NEXT:    add.s16 %rs6, %rs5, 1;
; SM70-NEXT:    prmt.b32 %r6, %r34, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r6;
; SM70-NEXT:    add.s16 %rs8, %rs7, 1;
; SM70-NEXT:    prmt.b32 %r7, %r33, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs9, %r7;
; SM70-NEXT:    add.s16 %rs10, %rs9, 1;
; SM70-NEXT:    prmt.b32 %r8, %r33, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r8;
; SM70-NEXT:    add.s16 %rs12, %rs11, 1;
; SM70-NEXT:    prmt.b32 %r9, %r33, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r9;
; SM70-NEXT:    add.s16 %rs14, %rs13, 1;
; SM70-NEXT:    prmt.b32 %r10, %r33, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs15, %r10;
; SM70-NEXT:    add.s16 %rs16, %rs15, 1;
; SM70-NEXT:    prmt.b32 %r11, %r2, 0, 0x7770U;
; SM70-NEXT:    setp.ge.u32 %p1, %r3, %r11;
; SM70-NEXT:    prmt.b32 %r12, %r2, 0, 0x7771U;
; SM70-NEXT:    setp.ge.u32 %p2, %r4, %r12;
; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7772U;
; SM70-NEXT:    setp.ge.u32 %p3, %r5, %r13;
; SM70-NEXT:    prmt.b32 %r14, %r2, 0, 0x7773U;
; SM70-NEXT:    setp.ge.u32 %p4, %r6, %r14;
; SM70-NEXT:    prmt.b32 %r15, %r1, 0, 0x7770U;
; SM70-NEXT:    setp.ge.u32 %p5, %r7, %r15;
; SM70-NEXT:    prmt.b32 %r16, %r1, 0, 0x7771U;
; SM70-NEXT:    setp.ge.u32 %p6, %r8, %r16;
; SM70-NEXT:    prmt.b32 %r17, %r1, 0, 0x7772U;
; SM70-NEXT:    setp.ge.u32 %p7, %r9, %r17;
; SM70-NEXT:    prmt.b32 %r18, %r1, 0, 0x7773U;
; SM70-NEXT:    setp.ge.u32 %p8, %r10, %r18;
; SM70-NEXT:    selp.b16 %rs17, 0, %rs16, %p8;
; SM70-NEXT:    cvt.u32.u16 %r19, %rs17;
; SM70-NEXT:    selp.b16 %rs18, 0, %rs14, %p7;
; SM70-NEXT:    cvt.u32.u16 %r20, %rs18;
; SM70-NEXT:    prmt.b32 %r21, %r20, %r19, 0x3340U;
; SM70-NEXT:    selp.b16 %rs19, 0, %rs12, %p6;
; SM70-NEXT:    cvt.u32.u16 %r22, %rs19;
; SM70-NEXT:    selp.b16 %rs20, 0, %rs10, %p5;
; SM70-NEXT:    cvt.u32.u16 %r23, %rs20;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r22, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r24, %r21, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r25;
; SM70-NEXT:    selp.b16 %rs21, 0, %rs8, %p4;
; SM70-NEXT:    cvt.u32.u16 %r26, %rs21;
; SM70-NEXT:    selp.b16 %rs22, 0, %rs6, %p3;
; SM70-NEXT:    cvt.u32.u16 %r27, %rs22;
; SM70-NEXT:    prmt.b32 %r28, %r27, %r26, 0x3340U;
; SM70-NEXT:    selp.b16 %rs23, 0, %rs4, %p2;
; SM70-NEXT:    cvt.u32.u16 %r29, %rs23;
; SM70-NEXT:    selp.b16 %rs24, 0, %rs2, %p1;
; SM70-NEXT:    cvt.u32.u16 %r30, %rs24;
; SM70-NEXT:    prmt.b32 %r31, %r30, %r29, 0x3340U;
; SM70-NEXT:    prmt.b32 %r32, %r31, %r28, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r32;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NEXT:    @%p9 bra $L__BB179_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @uinc_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b16 %rs<5>;
; SM70-NEXT:    .reg .b32 %r<14>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [uinc_wrap_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %rs1, [uinc_wrap_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NEXT:  $L__BB180_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    add.s16 %rs3, %rs2, 1;
; SM70-NEXT:    setp.ge.u16 %p1, %rs2, %rs1;
; SM70-NEXT:    selp.b16 %rs4, 0, %rs3, %p1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs4;
; SM70-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NEXT:    setp.ne.b32 %p2, %r3, %r13;
; SM70-NEXT:    mov.b32 %r13, %r3;
; SM70-NEXT:    @%p2 bra $L__BB180_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @uinc_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<4>;
; SM70-NEXT:    .reg .b16 %rs<9>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [uinc_wrap_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NEXT:  $L__BB181_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
; SM70-NEXT:    add.s16 %rs3, %rs1, 1;
; SM70-NEXT:    add.s16 %rs4, %rs2, 1;
; SM70-NEXT:    setp.ge.u16 %p1, %rs1, %rs5;
; SM70-NEXT:    setp.ge.u16 %p2, %rs2, %rs6;
; SM70-NEXT:    selp.b16 %rs7, 0, %rs4, %p2;
; SM70-NEXT:    selp.b16 %rs8, 0, %rs3, %p1;
; SM70-NEXT:    mov.b32 %r3, {%rs8, %rs7};
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p3, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p3 bra $L__BB181_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @uinc_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<6>;
; SM70-NEXT:    .reg .b16 %rs<17>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
; SM70-NEXT:  $L__BB182_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r6;
; SM70-NEXT:    add.s16 %rs3, %rs1, 1;
; SM70-NEXT:    add.s16 %rs4, %rs2, 1;
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r5;
; SM70-NEXT:    add.s16 %rs7, %rs5, 1;
; SM70-NEXT:    add.s16 %rs8, %rs6, 1;
; SM70-NEXT:    setp.ge.u16 %p1, %rs1, %rs9;
; SM70-NEXT:    setp.ge.u16 %p2, %rs2, %rs10;
; SM70-NEXT:    setp.ge.u16 %p3, %rs5, %rs11;
; SM70-NEXT:    setp.ge.u16 %p4, %rs6, %rs12;
; SM70-NEXT:    selp.b16 %rs13, 0, %rs8, %p4;
; SM70-NEXT:    selp.b16 %rs14, 0, %rs7, %p3;
; SM70-NEXT:    mov.b32 %r3, {%rs14, %rs13};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r3;
; SM70-NEXT:    selp.b16 %rs15, 0, %rs4, %p2;
; SM70-NEXT:    selp.b16 %rs16, 0, %rs3, %p1;
; SM70-NEXT:    mov.b32 %r4, {%rs16, %rs15};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r4;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p5, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p5 bra $L__BB182_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @uinc_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<11>;
; SM70-NEXT:    .reg .b16 %rs<33>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
; SM70-NEXT:  $L__BB183_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r10;
; SM70-NEXT:    add.s16 %rs3, %rs1, 1;
; SM70-NEXT:    add.s16 %rs4, %rs2, 1;
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r9;
; SM70-NEXT:    add.s16 %rs7, %rs5, 1;
; SM70-NEXT:    add.s16 %rs8, %rs6, 1;
; SM70-NEXT:    setp.ge.u16 %p1, %rs1, %rs9;
; SM70-NEXT:    setp.ge.u16 %p2, %rs2, %rs10;
; SM70-NEXT:    setp.ge.u16 %p3, %rs5, %rs11;
; SM70-NEXT:    setp.ge.u16 %p4, %rs6, %rs12;
; SM70-NEXT:    selp.b16 %rs13, 0, %rs8, %p4;
; SM70-NEXT:    selp.b16 %rs14, 0, %rs7, %p3;
; SM70-NEXT:    mov.b32 %r5, {%rs14, %rs13};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    selp.b16 %rs15, 0, %rs4, %p2;
; SM70-NEXT:    selp.b16 %rs16, 0, %rs3, %p1;
; SM70-NEXT:    mov.b32 %r6, {%rs16, %rs15};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p5, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NEXT:    @%p5 bra $L__BB183_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NEXT:  $L__BB183_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    mov.b32 {%rs17, %rs18}, %r12;
; SM70-NEXT:    add.s16 %rs19, %rs17, 1;
; SM70-NEXT:    add.s16 %rs20, %rs18, 1;
; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r11;
; SM70-NEXT:    add.s16 %rs23, %rs21, 1;
; SM70-NEXT:    add.s16 %rs24, %rs22, 1;
; SM70-NEXT:    mov.b32 {%rs25, %rs26}, %r4;
; SM70-NEXT:    setp.ge.u16 %p6, %rs17, %rs25;
; SM70-NEXT:    setp.ge.u16 %p7, %rs18, %rs26;
; SM70-NEXT:    mov.b32 {%rs27, %rs28}, %r3;
; SM70-NEXT:    setp.ge.u16 %p8, %rs21, %rs27;
; SM70-NEXT:    setp.ge.u16 %p9, %rs22, %rs28;
; SM70-NEXT:    selp.b16 %rs29, 0, %rs24, %p9;
; SM70-NEXT:    selp.b16 %rs30, 0, %rs23, %p8;
; SM70-NEXT:    mov.b32 %r7, {%rs30, %rs29};
; SM70-NEXT:    cvt.u64.u32 %rd17, %r7;
; SM70-NEXT:    selp.b16 %rs31, 0, %rs20, %p7;
; SM70-NEXT:    selp.b16 %rs32, 0, %rs19, %p6;
; SM70-NEXT:    mov.b32 %r8, {%rs32, %rs31};
; SM70-NEXT:    cvt.u64.u32 %rd18, %r8;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
; SM70-NEXT:    setp.ne.b64 %p10, %rd21, %rd16;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NEXT:    @%p10 bra $L__BB183_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @uinc_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [uinc_wrap_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r2, [%rd1], %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @uinc_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [uinc_wrap_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r3, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r4, [%rd1+4], %r2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @uinc_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r5, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r6, [%rd1+4], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r7, [%rd1+8], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r8, [%rd1+12], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @uinc_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [uinc_wrap_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r9, [%rd1], %r5;
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r10, [%rd1+4], %r6;
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r11, [%rd1+8], %r7;
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r12, [%rd1+12], %r8;
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r13, [%rd1+16], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r14, [%rd1+20], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r15, [%rd1+24], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.inc.u32 %r16, [%rd1+28], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @uinc_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b64 %rd<7>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd3, [uinc_wrap_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd1, [uinc_wrap_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd6, [%rd3];
; SM70-NEXT:  $L__BB188_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    add.s64 %rd4, %rd6, 1;
; SM70-NEXT:    setp.ge.u64 %p1, %rd6, %rd1;
; SM70-NEXT:    selp.b64 %rd5, 0, %rd4, %p1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
; SM70-NEXT:    setp.ne.b64 %p2, %rd2, %rd6;
; SM70-NEXT:    mov.b64 %rd6, %rd2;
; SM70-NEXT:    @%p2 bra $L__BB188_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @uinc_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<5>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd5, [uinc_wrap_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd1}, [uinc_wrap_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd10, [%rd5];
; SM70-NEXT:  $L__BB189_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd3, %rd10;
; SM70-NEXT:    add.s64 %rd6, %rd3, 1;
; SM70-NEXT:    setp.ge.u64 %p1, %rd3, %rd2;
; SM70-NEXT:    selp.b64 %rd7, 0, %rd6, %p1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd10, [%rd5], %rd3, %rd7;
; SM70-NEXT:    setp.ne.b64 %p2, %rd10, %rd3;
; SM70-NEXT:    @%p2 bra $L__BB189_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
; SM70-NEXT:  $L__BB189_3: // %atomicrmw.start4
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    add.s64 %rd8, %rd11, 1;
; SM70-NEXT:    setp.ge.u64 %p3, %rd11, %rd1;
; SM70-NEXT:    selp.b64 %rd9, 0, %rd8, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM70-NEXT:    setp.ne.b64 %p4, %rd4, %rd11;
; SM70-NEXT:    mov.b64 %rd11, %rd4;
; SM70-NEXT:    @%p4 bra $L__BB189_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end3
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @uinc_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<9>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd9, [uinc_wrap_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd6}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd3, %rd2}, [uinc_wrap_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd18, [%rd9];
; SM70-NEXT:  $L__BB190_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd4, %rd18;
; SM70-NEXT:    add.s64 %rd10, %rd4, 1;
; SM70-NEXT:    setp.ge.u64 %p1, %rd4, %rd3;
; SM70-NEXT:    selp.b64 %rd11, 0, %rd10, %p1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd4, %rd11;
; SM70-NEXT:    setp.ne.b64 %p2, %rd18, %rd4;
; SM70-NEXT:    @%p2 bra $L__BB190_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
; SM70-NEXT:  $L__BB190_3: // %atomicrmw.start8
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd19;
; SM70-NEXT:    add.s64 %rd12, %rd5, 1;
; SM70-NEXT:    setp.ge.u64 %p3, %rd5, %rd2;
; SM70-NEXT:    selp.b64 %rd13, 0, %rd12, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd5, %rd13;
; SM70-NEXT:    setp.ne.b64 %p4, %rd19, %rd5;
; SM70-NEXT:    @%p4 bra $L__BB190_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end7
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
; SM70-NEXT:  $L__BB190_5: // %atomicrmw.start20
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd7, %rd20;
; SM70-NEXT:    add.s64 %rd14, %rd7, 1;
; SM70-NEXT:    setp.ge.u64 %p5, %rd7, %rd1;
; SM70-NEXT:    selp.b64 %rd15, 0, %rd14, %p5;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd20, [%rd9+16], %rd7, %rd15;
; SM70-NEXT:    setp.ne.b64 %p6, %rd20, %rd7;
; SM70-NEXT:    @%p6 bra $L__BB190_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end19
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
; SM70-NEXT:  $L__BB190_7: // %atomicrmw.start28
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    add.s64 %rd16, %rd21, 1;
; SM70-NEXT:    setp.ge.u64 %p7, %rd21, %rd6;
; SM70-NEXT:    selp.b64 %rd17, 0, %rd16, %p7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM70-NEXT:    setp.ne.b64 %p8, %rd8, %rd21;
; SM70-NEXT:    mov.b64 %rd21, %rd8;
; SM70-NEXT:    @%p8 bra $L__BB190_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end27
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd20, %rd8};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @uinc_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: uinc_wrap_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<17>;
; SM70-NEXT:    .reg .b64 %rd<42>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd17, [uinc_wrap_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd10, %rd14}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd11}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd7}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd3}, [uinc_wrap_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd34, [%rd17];
; SM70-NEXT:  $L__BB191_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd34;
; SM70-NEXT:    add.s64 %rd18, %rd5, 1;
; SM70-NEXT:    setp.ge.u64 %p1, %rd5, %rd4;
; SM70-NEXT:    selp.b64 %rd19, 0, %rd18, %p1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd5, %rd19;
; SM70-NEXT:    setp.ne.b64 %p2, %rd34, %rd5;
; SM70-NEXT:    @%p2 bra $L__BB191_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
; SM70-NEXT:  $L__BB191_3: // %atomicrmw.start12
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd6, %rd35;
; SM70-NEXT:    add.s64 %rd20, %rd6, 1;
; SM70-NEXT:    setp.ge.u64 %p3, %rd6, %rd3;
; SM70-NEXT:    selp.b64 %rd21, 0, %rd20, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd6, %rd21;
; SM70-NEXT:    setp.ne.b64 %p4, %rd35, %rd6;
; SM70-NEXT:    @%p4 bra $L__BB191_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end11
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
; SM70-NEXT:  $L__BB191_5: // %atomicrmw.start24
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd8, %rd36;
; SM70-NEXT:    add.s64 %rd22, %rd8, 1;
; SM70-NEXT:    setp.ge.u64 %p5, %rd8, %rd2;
; SM70-NEXT:    selp.b64 %rd23, 0, %rd22, %p5;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd8, %rd23;
; SM70-NEXT:    setp.ne.b64 %p6, %rd36, %rd8;
; SM70-NEXT:    @%p6 bra $L__BB191_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end23
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
; SM70-NEXT:  $L__BB191_7: // %atomicrmw.start32
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd9, %rd37;
; SM70-NEXT:    add.s64 %rd24, %rd9, 1;
; SM70-NEXT:    setp.ge.u64 %p7, %rd9, %rd7;
; SM70-NEXT:    selp.b64 %rd25, 0, %rd24, %p7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd9, %rd25;
; SM70-NEXT:    setp.ne.b64 %p8, %rd37, %rd9;
; SM70-NEXT:    @%p8 bra $L__BB191_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end31
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
; SM70-NEXT:  $L__BB191_9: // %atomicrmw.start48
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd12, %rd38;
; SM70-NEXT:    add.s64 %rd26, %rd12, 1;
; SM70-NEXT:    setp.ge.u64 %p9, %rd12, %rd1;
; SM70-NEXT:    selp.b64 %rd27, 0, %rd26, %p9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd12, %rd27;
; SM70-NEXT:    setp.ne.b64 %p10, %rd38, %rd12;
; SM70-NEXT:    @%p10 bra $L__BB191_9;
; SM70-NEXT:  // %bb.10: // %atomicrmw.end47
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
; SM70-NEXT:  $L__BB191_11: // %atomicrmw.start56
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd13, %rd39;
; SM70-NEXT:    add.s64 %rd28, %rd13, 1;
; SM70-NEXT:    setp.ge.u64 %p11, %rd13, %rd11;
; SM70-NEXT:    selp.b64 %rd29, 0, %rd28, %p11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd13, %rd29;
; SM70-NEXT:    setp.ne.b64 %p12, %rd39, %rd13;
; SM70-NEXT:    @%p12 bra $L__BB191_11;
; SM70-NEXT:  // %bb.12: // %atomicrmw.end55
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
; SM70-NEXT:  $L__BB191_13: // %atomicrmw.start68
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd15, %rd40;
; SM70-NEXT:    add.s64 %rd30, %rd15, 1;
; SM70-NEXT:    setp.ge.u64 %p13, %rd15, %rd10;
; SM70-NEXT:    selp.b64 %rd31, 0, %rd30, %p13;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd40, [%rd17+48], %rd15, %rd31;
; SM70-NEXT:    setp.ne.b64 %p14, %rd40, %rd15;
; SM70-NEXT:    @%p14 bra $L__BB191_13;
; SM70-NEXT:  // %bb.14: // %atomicrmw.end67
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
; SM70-NEXT:  $L__BB191_15: // %atomicrmw.start76
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    add.s64 %rd32, %rd41, 1;
; SM70-NEXT:    setp.ge.u64 %p15, %rd41, %rd14;
; SM70-NEXT:    selp.b64 %rd33, 0, %rd32, %p15;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM70-NEXT:    setp.ne.b64 %p16, %rd16, %rd41;
; SM70-NEXT:    mov.b64 %rd41, %rd16;
; SM70-NEXT:    @%p16 bra $L__BB191_15;
; SM70-NEXT:  // %bb.16: // %atomicrmw.end75
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd40, %rd16};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd34, %rd35};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise uinc_wrap ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @udec_wrap_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<4>;
; SM70-NEXT:    .reg .b16 %rs<7>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b8 %rs1, [udec_wrap_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 255, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:  $L__BB192_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r14, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    and.b16 %rs3, %rs2, 255;
; SM70-NEXT:    add.s16 %rs4, %rs2, -1;
; SM70-NEXT:    setp.eq.b16 %p1, %rs3, 0;
; SM70-NEXT:    setp.gt.u16 %p2, %rs3, %rs1;
; SM70-NEXT:    selp.b16 %rs5, %rs1, %rs4, %p2;
; SM70-NEXT:    selp.b16 %rs6, %rs1, %rs5, %p1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NEXT:    and.b32 %r9, %r8, 255;
; SM70-NEXT:    shl.b32 %r10, %r9, %r1;
; SM70-NEXT:    and.b32 %r11, %r14, %r2;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p3, %r3, %r14;
; SM70-NEXT:    mov.b32 %r14, %r3;
; SM70-NEXT:    @%p3 bra $L__BB192_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @udec_wrap_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<6>;
; SM70-NEXT:    .reg .b16 %rs<17>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [udec_wrap_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT:    ld.param.b64 %rd2, [udec_wrap_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    and.b32 %r6, %r5, 3;
; SM70-NEXT:    shl.b32 %r1, %r6, 3;
; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r7;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:    mov.b32 {%rs8, %rs9}, %r4;
; SM70-NEXT:  $L__BB193_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r8, %r14, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r8;
; SM70-NEXT:    and.b16 %rs4, %rs3, 255;
; SM70-NEXT:    shr.u16 %rs5, %rs3, 8;
; SM70-NEXT:    add.s16 %rs6, %rs5, -1;
; SM70-NEXT:    add.s16 %rs7, %rs3, -1;
; SM70-NEXT:    setp.eq.b16 %p1, %rs5, 0;
; SM70-NEXT:    setp.eq.b16 %p2, %rs4, 0;
; SM70-NEXT:    setp.gt.u16 %p3, %rs5, %rs9;
; SM70-NEXT:    setp.gt.u16 %p4, %rs4, %rs8;
; SM70-NEXT:    selp.b16 %rs10, %rs8, %rs7, %p4;
; SM70-NEXT:    selp.b16 %rs11, %rs8, %rs10, %p2;
; SM70-NEXT:    selp.b16 %rs12, %rs9, %rs6, %p3;
; SM70-NEXT:    selp.b16 %rs13, %rs9, %rs12, %p1;
; SM70-NEXT:    shl.b16 %rs14, %rs13, 8;
; SM70-NEXT:    and.b16 %rs15, %rs11, 255;
; SM70-NEXT:    or.b16 %rs16, %rs15, %rs14;
; SM70-NEXT:    cvt.u32.u16 %r9, %rs16;
; SM70-NEXT:    shl.b32 %r10, %r9, %r1;
; SM70-NEXT:    and.b32 %r11, %r14, %r2;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p5, %r3, %r14;
; SM70-NEXT:    mov.b32 %r14, %r3;
; SM70-NEXT:    @%p5 bra $L__BB193_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @udec_wrap_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<10>;
; SM70-NEXT:    .reg .b16 %rs<9>;
; SM70-NEXT:    .reg .b32 %r<27>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [udec_wrap_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7770U;
; SM70-NEXT:    prmt.b32 %r8, %r2, 0, 0x7771U;
; SM70-NEXT:  $L__BB194_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r3, %r26, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    add.s16 %rs2, %rs1, -1;
; SM70-NEXT:    prmt.b32 %r4, %r26, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r4;
; SM70-NEXT:    add.s16 %rs4, %rs3, -1;
; SM70-NEXT:    prmt.b32 %r5, %r26, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r5;
; SM70-NEXT:    add.s16 %rs6, %rs5, -1;
; SM70-NEXT:    prmt.b32 %r6, %r26, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r6;
; SM70-NEXT:    add.s16 %rs8, %rs7, -1;
; SM70-NEXT:    setp.eq.b32 %p1, %r3, 0;
; SM70-NEXT:    setp.eq.b32 %p2, %r4, 0;
; SM70-NEXT:    setp.eq.b32 %p3, %r5, 0;
; SM70-NEXT:    setp.eq.b32 %p4, %r6, 0;
; SM70-NEXT:    setp.gt.u32 %p5, %r3, %r7;
; SM70-NEXT:    setp.gt.u32 %p6, %r4, %r8;
; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0x7772U;
; SM70-NEXT:    setp.gt.u32 %p7, %r5, %r9;
; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7773U;
; SM70-NEXT:    setp.gt.u32 %p8, %r6, %r10;
; SM70-NEXT:    cvt.u32.u16 %r11, %rs8;
; SM70-NEXT:    selp.b32 %r12, %r10, %r11, %p8;
; SM70-NEXT:    selp.b32 %r13, %r10, %r12, %p4;
; SM70-NEXT:    cvt.u32.u16 %r14, %rs6;
; SM70-NEXT:    selp.b32 %r15, %r9, %r14, %p7;
; SM70-NEXT:    selp.b32 %r16, %r9, %r15, %p3;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r13, 0x3340U;
; SM70-NEXT:    cvt.u32.u16 %r18, %rs4;
; SM70-NEXT:    selp.b32 %r19, %r8, %r18, %p6;
; SM70-NEXT:    selp.b32 %r20, %r8, %r19, %p2;
; SM70-NEXT:    cvt.u32.u16 %r21, %rs2;
; SM70-NEXT:    selp.b32 %r22, %r7, %r21, %p5;
; SM70-NEXT:    selp.b32 %r23, %r7, %r22, %p1;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r20, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r24, %r17, 0x5410U;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NEXT:    setp.ne.b32 %p9, %r1, %r26;
; SM70-NEXT:    mov.b32 %r26, %r1;
; SM70-NEXT:    @%p9 bra $L__BB194_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @udec_wrap_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<18>;
; SM70-NEXT:    .reg .b16 %rs<41>;
; SM70-NEXT:    .reg .b32 %r<35>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NEXT:  $L__BB195_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r33;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r34;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    prmt.b32 %r3, %r34, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    add.s16 %rs2, %rs1, -1;
; SM70-NEXT:    prmt.b32 %r4, %r34, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r4;
; SM70-NEXT:    add.s16 %rs4, %rs3, -1;
; SM70-NEXT:    prmt.b32 %r5, %r34, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r5;
; SM70-NEXT:    add.s16 %rs6, %rs5, -1;
; SM70-NEXT:    prmt.b32 %r6, %r34, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r6;
; SM70-NEXT:    add.s16 %rs8, %rs7, -1;
; SM70-NEXT:    prmt.b32 %r7, %r33, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs9, %r7;
; SM70-NEXT:    add.s16 %rs10, %rs9, -1;
; SM70-NEXT:    prmt.b32 %r8, %r33, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r8;
; SM70-NEXT:    add.s16 %rs12, %rs11, -1;
; SM70-NEXT:    prmt.b32 %r9, %r33, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r9;
; SM70-NEXT:    add.s16 %rs14, %rs13, -1;
; SM70-NEXT:    prmt.b32 %r10, %r33, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs15, %r10;
; SM70-NEXT:    add.s16 %rs16, %rs15, -1;
; SM70-NEXT:    setp.eq.b32 %p1, %r3, 0;
; SM70-NEXT:    setp.eq.b32 %p2, %r4, 0;
; SM70-NEXT:    setp.eq.b32 %p3, %r5, 0;
; SM70-NEXT:    setp.eq.b32 %p4, %r6, 0;
; SM70-NEXT:    setp.eq.b32 %p5, %r7, 0;
; SM70-NEXT:    setp.eq.b32 %p6, %r8, 0;
; SM70-NEXT:    setp.eq.b32 %p7, %r9, 0;
; SM70-NEXT:    setp.eq.b32 %p8, %r10, 0;
; SM70-NEXT:    prmt.b32 %r11, %r2, 0, 0x7770U;
; SM70-NEXT:    setp.gt.u32 %p9, %r3, %r11;
; SM70-NEXT:    prmt.b32 %r12, %r2, 0, 0x7771U;
; SM70-NEXT:    setp.gt.u32 %p10, %r4, %r12;
; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7772U;
; SM70-NEXT:    setp.gt.u32 %p11, %r5, %r13;
; SM70-NEXT:    prmt.b32 %r14, %r2, 0, 0x7773U;
; SM70-NEXT:    setp.gt.u32 %p12, %r6, %r14;
; SM70-NEXT:    prmt.b32 %r15, %r1, 0, 0x7770U;
; SM70-NEXT:    setp.gt.u32 %p13, %r7, %r15;
; SM70-NEXT:    prmt.b32 %r16, %r1, 0, 0x7771U;
; SM70-NEXT:    setp.gt.u32 %p14, %r8, %r16;
; SM70-NEXT:    prmt.b32 %r17, %r1, 0, 0x7772U;
; SM70-NEXT:    setp.gt.u32 %p15, %r9, %r17;
; SM70-NEXT:    prmt.b32 %r18, %r1, 0, 0x7773U;
; SM70-NEXT:    setp.gt.u32 %p16, %r10, %r18;
; SM70-NEXT:    cvt.u16.u32 %rs17, %r11;
; SM70-NEXT:    cvt.u16.u32 %rs18, %r12;
; SM70-NEXT:    cvt.u16.u32 %rs19, %r13;
; SM70-NEXT:    cvt.u16.u32 %rs20, %r14;
; SM70-NEXT:    cvt.u16.u32 %rs21, %r15;
; SM70-NEXT:    cvt.u16.u32 %rs22, %r16;
; SM70-NEXT:    cvt.u16.u32 %rs23, %r17;
; SM70-NEXT:    cvt.u16.u32 %rs24, %r18;
; SM70-NEXT:    selp.b16 %rs25, %rs24, %rs16, %p16;
; SM70-NEXT:    selp.b16 %rs26, %rs24, %rs25, %p8;
; SM70-NEXT:    cvt.u32.u16 %r19, %rs26;
; SM70-NEXT:    selp.b16 %rs27, %rs23, %rs14, %p15;
; SM70-NEXT:    selp.b16 %rs28, %rs23, %rs27, %p7;
; SM70-NEXT:    cvt.u32.u16 %r20, %rs28;
; SM70-NEXT:    prmt.b32 %r21, %r20, %r19, 0x3340U;
; SM70-NEXT:    selp.b16 %rs29, %rs22, %rs12, %p14;
; SM70-NEXT:    selp.b16 %rs30, %rs22, %rs29, %p6;
; SM70-NEXT:    cvt.u32.u16 %r22, %rs30;
; SM70-NEXT:    selp.b16 %rs31, %rs21, %rs10, %p13;
; SM70-NEXT:    selp.b16 %rs32, %rs21, %rs31, %p5;
; SM70-NEXT:    cvt.u32.u16 %r23, %rs32;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r22, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r24, %r21, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r25;
; SM70-NEXT:    selp.b16 %rs33, %rs20, %rs8, %p12;
; SM70-NEXT:    selp.b16 %rs34, %rs20, %rs33, %p4;
; SM70-NEXT:    cvt.u32.u16 %r26, %rs34;
; SM70-NEXT:    selp.b16 %rs35, %rs19, %rs6, %p11;
; SM70-NEXT:    selp.b16 %rs36, %rs19, %rs35, %p3;
; SM70-NEXT:    cvt.u32.u16 %r27, %rs36;
; SM70-NEXT:    prmt.b32 %r28, %r27, %r26, 0x3340U;
; SM70-NEXT:    selp.b16 %rs37, %rs18, %rs4, %p10;
; SM70-NEXT:    selp.b16 %rs38, %rs18, %rs37, %p2;
; SM70-NEXT:    cvt.u32.u16 %r29, %rs38;
; SM70-NEXT:    selp.b16 %rs39, %rs17, %rs2, %p9;
; SM70-NEXT:    selp.b16 %rs40, %rs17, %rs39, %p1;
; SM70-NEXT:    cvt.u32.u16 %r30, %rs40;
; SM70-NEXT:    prmt.b32 %r31, %r30, %r29, 0x3340U;
; SM70-NEXT:    prmt.b32 %r32, %r31, %r28, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r32;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p17, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NEXT:    @%p17 bra $L__BB195_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @udec_wrap_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<4>;
; SM70-NEXT:    .reg .b16 %rs<6>;
; SM70-NEXT:    .reg .b32 %r<14>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [udec_wrap_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %rs1, [udec_wrap_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NEXT:  $L__BB196_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    add.s16 %rs3, %rs2, -1;
; SM70-NEXT:    setp.eq.b16 %p1, %rs2, 0;
; SM70-NEXT:    setp.gt.u16 %p2, %rs2, %rs1;
; SM70-NEXT:    selp.b16 %rs4, %rs1, %rs3, %p2;
; SM70-NEXT:    selp.b16 %rs5, %rs1, %rs4, %p1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs5;
; SM70-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NEXT:    setp.ne.b32 %p3, %r3, %r13;
; SM70-NEXT:    mov.b32 %r13, %r3;
; SM70-NEXT:    @%p3 bra $L__BB196_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @udec_wrap_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<6>;
; SM70-NEXT:    .reg .b16 %rs<11>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [udec_wrap_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NEXT:  $L__BB197_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
; SM70-NEXT:    add.s16 %rs3, %rs1, -1;
; SM70-NEXT:    add.s16 %rs4, %rs2, -1;
; SM70-NEXT:    setp.eq.b16 %p1, %rs1, 0;
; SM70-NEXT:    setp.eq.b16 %p2, %rs2, 0;
; SM70-NEXT:    setp.gt.u16 %p3, %rs1, %rs5;
; SM70-NEXT:    setp.gt.u16 %p4, %rs2, %rs6;
; SM70-NEXT:    selp.b16 %rs7, %rs6, %rs4, %p4;
; SM70-NEXT:    selp.b16 %rs8, %rs6, %rs7, %p2;
; SM70-NEXT:    selp.b16 %rs9, %rs5, %rs3, %p3;
; SM70-NEXT:    selp.b16 %rs10, %rs5, %rs9, %p1;
; SM70-NEXT:    mov.b32 %r3, {%rs10, %rs8};
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p5, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p5 bra $L__BB197_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @udec_wrap_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<10>;
; SM70-NEXT:    .reg .b16 %rs<21>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
; SM70-NEXT:  $L__BB198_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r6;
; SM70-NEXT:    add.s16 %rs3, %rs1, -1;
; SM70-NEXT:    add.s16 %rs4, %rs2, -1;
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r5;
; SM70-NEXT:    add.s16 %rs7, %rs5, -1;
; SM70-NEXT:    add.s16 %rs8, %rs6, -1;
; SM70-NEXT:    setp.eq.b16 %p1, %rs1, 0;
; SM70-NEXT:    setp.eq.b16 %p2, %rs2, 0;
; SM70-NEXT:    setp.eq.b16 %p3, %rs5, 0;
; SM70-NEXT:    setp.eq.b16 %p4, %rs6, 0;
; SM70-NEXT:    setp.gt.u16 %p5, %rs1, %rs9;
; SM70-NEXT:    setp.gt.u16 %p6, %rs2, %rs10;
; SM70-NEXT:    setp.gt.u16 %p7, %rs5, %rs11;
; SM70-NEXT:    setp.gt.u16 %p8, %rs6, %rs12;
; SM70-NEXT:    selp.b16 %rs13, %rs12, %rs8, %p8;
; SM70-NEXT:    selp.b16 %rs14, %rs12, %rs13, %p4;
; SM70-NEXT:    selp.b16 %rs15, %rs11, %rs7, %p7;
; SM70-NEXT:    selp.b16 %rs16, %rs11, %rs15, %p3;
; SM70-NEXT:    mov.b32 %r3, {%rs16, %rs14};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r3;
; SM70-NEXT:    selp.b16 %rs17, %rs10, %rs4, %p6;
; SM70-NEXT:    selp.b16 %rs18, %rs10, %rs17, %p2;
; SM70-NEXT:    selp.b16 %rs19, %rs9, %rs3, %p5;
; SM70-NEXT:    selp.b16 %rs20, %rs9, %rs19, %p1;
; SM70-NEXT:    mov.b32 %r4, {%rs20, %rs18};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r4;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p9 bra $L__BB198_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @udec_wrap_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<19>;
; SM70-NEXT:    .reg .b16 %rs<41>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r1;
; SM70-NEXT:  $L__BB199_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r10;
; SM70-NEXT:    add.s16 %rs3, %rs1, -1;
; SM70-NEXT:    add.s16 %rs4, %rs2, -1;
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r9;
; SM70-NEXT:    add.s16 %rs7, %rs5, -1;
; SM70-NEXT:    add.s16 %rs8, %rs6, -1;
; SM70-NEXT:    setp.eq.b16 %p1, %rs1, 0;
; SM70-NEXT:    setp.eq.b16 %p2, %rs2, 0;
; SM70-NEXT:    setp.eq.b16 %p3, %rs5, 0;
; SM70-NEXT:    setp.eq.b16 %p4, %rs6, 0;
; SM70-NEXT:    setp.gt.u16 %p5, %rs1, %rs9;
; SM70-NEXT:    setp.gt.u16 %p6, %rs2, %rs10;
; SM70-NEXT:    setp.gt.u16 %p7, %rs5, %rs11;
; SM70-NEXT:    setp.gt.u16 %p8, %rs6, %rs12;
; SM70-NEXT:    selp.b16 %rs13, %rs12, %rs8, %p8;
; SM70-NEXT:    selp.b16 %rs14, %rs12, %rs13, %p4;
; SM70-NEXT:    selp.b16 %rs15, %rs11, %rs7, %p7;
; SM70-NEXT:    selp.b16 %rs16, %rs11, %rs15, %p3;
; SM70-NEXT:    mov.b32 %r5, {%rs16, %rs14};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    selp.b16 %rs17, %rs10, %rs4, %p6;
; SM70-NEXT:    selp.b16 %rs18, %rs10, %rs17, %p2;
; SM70-NEXT:    selp.b16 %rs19, %rs9, %rs3, %p5;
; SM70-NEXT:    selp.b16 %rs20, %rs9, %rs19, %p1;
; SM70-NEXT:    mov.b32 %r6, {%rs20, %rs18};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NEXT:    @%p9 bra $L__BB199_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NEXT:  $L__BB199_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r12;
; SM70-NEXT:    add.s16 %rs23, %rs21, -1;
; SM70-NEXT:    add.s16 %rs24, %rs22, -1;
; SM70-NEXT:    mov.b32 {%rs25, %rs26}, %r11;
; SM70-NEXT:    add.s16 %rs27, %rs25, -1;
; SM70-NEXT:    add.s16 %rs28, %rs26, -1;
; SM70-NEXT:    setp.eq.b16 %p10, %rs21, 0;
; SM70-NEXT:    setp.eq.b16 %p11, %rs22, 0;
; SM70-NEXT:    setp.eq.b16 %p12, %rs25, 0;
; SM70-NEXT:    setp.eq.b16 %p13, %rs26, 0;
; SM70-NEXT:    mov.b32 {%rs29, %rs30}, %r4;
; SM70-NEXT:    setp.gt.u16 %p14, %rs21, %rs29;
; SM70-NEXT:    setp.gt.u16 %p15, %rs22, %rs30;
; SM70-NEXT:    mov.b32 {%rs31, %rs32}, %r3;
; SM70-NEXT:    setp.gt.u16 %p16, %rs25, %rs31;
; SM70-NEXT:    setp.gt.u16 %p17, %rs26, %rs32;
; SM70-NEXT:    selp.b16 %rs33, %rs32, %rs28, %p17;
; SM70-NEXT:    selp.b16 %rs34, %rs32, %rs33, %p13;
; SM70-NEXT:    selp.b16 %rs35, %rs31, %rs27, %p16;
; SM70-NEXT:    selp.b16 %rs36, %rs31, %rs35, %p12;
; SM70-NEXT:    mov.b32 %r7, {%rs36, %rs34};
; SM70-NEXT:    cvt.u64.u32 %rd17, %r7;
; SM70-NEXT:    selp.b16 %rs37, %rs30, %rs24, %p15;
; SM70-NEXT:    selp.b16 %rs38, %rs30, %rs37, %p11;
; SM70-NEXT:    selp.b16 %rs39, %rs29, %rs23, %p14;
; SM70-NEXT:    selp.b16 %rs40, %rs29, %rs39, %p10;
; SM70-NEXT:    mov.b32 %r8, {%rs40, %rs38};
; SM70-NEXT:    cvt.u64.u32 %rd18, %r8;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
; SM70-NEXT:    setp.ne.b64 %p18, %rd21, %rd16;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NEXT:    @%p18 bra $L__BB199_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @udec_wrap_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [udec_wrap_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r2, [%rd1], %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @udec_wrap_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [udec_wrap_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r3, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r4, [%rd1+4], %r2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @udec_wrap_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r5, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r6, [%rd1+4], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r7, [%rd1+8], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r8, [%rd1+12], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @udec_wrap_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [udec_wrap_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [udec_wrap_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r9, [%rd1], %r5;
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r10, [%rd1+4], %r6;
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r11, [%rd1+8], %r7;
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r12, [%rd1+12], %r8;
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r13, [%rd1+16], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r14, [%rd1+20], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r15, [%rd1+24], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.dec.u32 %r16, [%rd1+28], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @udec_wrap_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<4>;
; SM70-NEXT:    .reg .b64 %rd<8>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd3, [udec_wrap_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd1, [udec_wrap_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd7, [%rd3];
; SM70-NEXT:  $L__BB204_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    add.s64 %rd4, %rd7, -1;
; SM70-NEXT:    setp.eq.b64 %p1, %rd7, 0;
; SM70-NEXT:    setp.gt.u64 %p2, %rd7, %rd1;
; SM70-NEXT:    selp.b64 %rd5, %rd1, %rd4, %p2;
; SM70-NEXT:    selp.b64 %rd6, %rd1, %rd5, %p1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd7, %rd6;
; SM70-NEXT:    setp.ne.b64 %p3, %rd2, %rd7;
; SM70-NEXT:    mov.b64 %rd7, %rd2;
; SM70-NEXT:    @%p3 bra $L__BB204_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @udec_wrap_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<7>;
; SM70-NEXT:    .reg .b64 %rd<14>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd5, [udec_wrap_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd1}, [udec_wrap_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd5];
; SM70-NEXT:  $L__BB205_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd3, %rd12;
; SM70-NEXT:    add.s64 %rd6, %rd3, -1;
; SM70-NEXT:    setp.eq.b64 %p1, %rd3, 0;
; SM70-NEXT:    setp.gt.u64 %p2, %rd3, %rd2;
; SM70-NEXT:    selp.b64 %rd7, %rd2, %rd6, %p2;
; SM70-NEXT:    selp.b64 %rd8, %rd2, %rd7, %p1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd12, [%rd5], %rd3, %rd8;
; SM70-NEXT:    setp.ne.b64 %p3, %rd12, %rd3;
; SM70-NEXT:    @%p3 bra $L__BB205_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd13, [%rd5+8];
; SM70-NEXT:  $L__BB205_3: // %atomicrmw.start4
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    add.s64 %rd9, %rd13, -1;
; SM70-NEXT:    setp.eq.b64 %p4, %rd13, 0;
; SM70-NEXT:    setp.gt.u64 %p5, %rd13, %rd1;
; SM70-NEXT:    selp.b64 %rd10, %rd1, %rd9, %p5;
; SM70-NEXT:    selp.b64 %rd11, %rd1, %rd10, %p4;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd13, %rd11;
; SM70-NEXT:    setp.ne.b64 %p6, %rd4, %rd13;
; SM70-NEXT:    mov.b64 %rd13, %rd4;
; SM70-NEXT:    @%p6 bra $L__BB205_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end3
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd12, %rd4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @udec_wrap_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<13>;
; SM70-NEXT:    .reg .b64 %rd<26>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd9, [udec_wrap_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd6}, [udec_wrap_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd3, %rd2}, [udec_wrap_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd9];
; SM70-NEXT:  $L__BB206_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd4, %rd22;
; SM70-NEXT:    add.s64 %rd10, %rd4, -1;
; SM70-NEXT:    setp.eq.b64 %p1, %rd4, 0;
; SM70-NEXT:    setp.gt.u64 %p2, %rd4, %rd3;
; SM70-NEXT:    selp.b64 %rd11, %rd3, %rd10, %p2;
; SM70-NEXT:    selp.b64 %rd12, %rd3, %rd11, %p1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd22, [%rd9], %rd4, %rd12;
; SM70-NEXT:    setp.ne.b64 %p3, %rd22, %rd4;
; SM70-NEXT:    @%p3 bra $L__BB206_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd23, [%rd9+8];
; SM70-NEXT:  $L__BB206_3: // %atomicrmw.start8
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd23;
; SM70-NEXT:    add.s64 %rd13, %rd5, -1;
; SM70-NEXT:    setp.eq.b64 %p4, %rd5, 0;
; SM70-NEXT:    setp.gt.u64 %p5, %rd5, %rd2;
; SM70-NEXT:    selp.b64 %rd14, %rd2, %rd13, %p5;
; SM70-NEXT:    selp.b64 %rd15, %rd2, %rd14, %p4;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd23, [%rd9+8], %rd5, %rd15;
; SM70-NEXT:    setp.ne.b64 %p6, %rd23, %rd5;
; SM70-NEXT:    @%p6 bra $L__BB206_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end7
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd24, [%rd9+16];
; SM70-NEXT:  $L__BB206_5: // %atomicrmw.start20
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd7, %rd24;
; SM70-NEXT:    add.s64 %rd16, %rd7, -1;
; SM70-NEXT:    setp.eq.b64 %p7, %rd7, 0;
; SM70-NEXT:    setp.gt.u64 %p8, %rd7, %rd1;
; SM70-NEXT:    selp.b64 %rd17, %rd1, %rd16, %p8;
; SM70-NEXT:    selp.b64 %rd18, %rd1, %rd17, %p7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd24, [%rd9+16], %rd7, %rd18;
; SM70-NEXT:    setp.ne.b64 %p9, %rd24, %rd7;
; SM70-NEXT:    @%p9 bra $L__BB206_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end19
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd25, [%rd9+24];
; SM70-NEXT:  $L__BB206_7: // %atomicrmw.start28
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    add.s64 %rd19, %rd25, -1;
; SM70-NEXT:    setp.eq.b64 %p10, %rd25, 0;
; SM70-NEXT:    setp.gt.u64 %p11, %rd25, %rd6;
; SM70-NEXT:    selp.b64 %rd20, %rd6, %rd19, %p11;
; SM70-NEXT:    selp.b64 %rd21, %rd6, %rd20, %p10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd25, %rd21;
; SM70-NEXT:    setp.ne.b64 %p12, %rd8, %rd25;
; SM70-NEXT:    mov.b64 %rd25, %rd8;
; SM70-NEXT:    @%p12 bra $L__BB206_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end27
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd24, %rd8};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd22, %rd23};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @udec_wrap_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: udec_wrap_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<25>;
; SM70-NEXT:    .reg .b64 %rd<50>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd17, [udec_wrap_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd10, %rd14}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd11}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd7}, [udec_wrap_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd3}, [udec_wrap_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd42, [%rd17];
; SM70-NEXT:  $L__BB207_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd42;
; SM70-NEXT:    add.s64 %rd18, %rd5, -1;
; SM70-NEXT:    setp.eq.b64 %p1, %rd5, 0;
; SM70-NEXT:    setp.gt.u64 %p2, %rd5, %rd4;
; SM70-NEXT:    selp.b64 %rd19, %rd4, %rd18, %p2;
; SM70-NEXT:    selp.b64 %rd20, %rd4, %rd19, %p1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd42, [%rd17], %rd5, %rd20;
; SM70-NEXT:    setp.ne.b64 %p3, %rd42, %rd5;
; SM70-NEXT:    @%p3 bra $L__BB207_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd43, [%rd17+8];
; SM70-NEXT:  $L__BB207_3: // %atomicrmw.start12
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd6, %rd43;
; SM70-NEXT:    add.s64 %rd21, %rd6, -1;
; SM70-NEXT:    setp.eq.b64 %p4, %rd6, 0;
; SM70-NEXT:    setp.gt.u64 %p5, %rd6, %rd3;
; SM70-NEXT:    selp.b64 %rd22, %rd3, %rd21, %p5;
; SM70-NEXT:    selp.b64 %rd23, %rd3, %rd22, %p4;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd43, [%rd17+8], %rd6, %rd23;
; SM70-NEXT:    setp.ne.b64 %p6, %rd43, %rd6;
; SM70-NEXT:    @%p6 bra $L__BB207_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end11
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd44, [%rd17+16];
; SM70-NEXT:  $L__BB207_5: // %atomicrmw.start24
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd8, %rd44;
; SM70-NEXT:    add.s64 %rd24, %rd8, -1;
; SM70-NEXT:    setp.eq.b64 %p7, %rd8, 0;
; SM70-NEXT:    setp.gt.u64 %p8, %rd8, %rd2;
; SM70-NEXT:    selp.b64 %rd25, %rd2, %rd24, %p8;
; SM70-NEXT:    selp.b64 %rd26, %rd2, %rd25, %p7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd44, [%rd17+16], %rd8, %rd26;
; SM70-NEXT:    setp.ne.b64 %p9, %rd44, %rd8;
; SM70-NEXT:    @%p9 bra $L__BB207_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end23
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd45, [%rd17+24];
; SM70-NEXT:  $L__BB207_7: // %atomicrmw.start32
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd9, %rd45;
; SM70-NEXT:    add.s64 %rd27, %rd9, -1;
; SM70-NEXT:    setp.eq.b64 %p10, %rd9, 0;
; SM70-NEXT:    setp.gt.u64 %p11, %rd9, %rd7;
; SM70-NEXT:    selp.b64 %rd28, %rd7, %rd27, %p11;
; SM70-NEXT:    selp.b64 %rd29, %rd7, %rd28, %p10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd45, [%rd17+24], %rd9, %rd29;
; SM70-NEXT:    setp.ne.b64 %p12, %rd45, %rd9;
; SM70-NEXT:    @%p12 bra $L__BB207_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end31
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd46, [%rd17+32];
; SM70-NEXT:  $L__BB207_9: // %atomicrmw.start48
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd12, %rd46;
; SM70-NEXT:    add.s64 %rd30, %rd12, -1;
; SM70-NEXT:    setp.eq.b64 %p13, %rd12, 0;
; SM70-NEXT:    setp.gt.u64 %p14, %rd12, %rd1;
; SM70-NEXT:    selp.b64 %rd31, %rd1, %rd30, %p14;
; SM70-NEXT:    selp.b64 %rd32, %rd1, %rd31, %p13;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd46, [%rd17+32], %rd12, %rd32;
; SM70-NEXT:    setp.ne.b64 %p15, %rd46, %rd12;
; SM70-NEXT:    @%p15 bra $L__BB207_9;
; SM70-NEXT:  // %bb.10: // %atomicrmw.end47
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd47, [%rd17+40];
; SM70-NEXT:  $L__BB207_11: // %atomicrmw.start56
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd13, %rd47;
; SM70-NEXT:    add.s64 %rd33, %rd13, -1;
; SM70-NEXT:    setp.eq.b64 %p16, %rd13, 0;
; SM70-NEXT:    setp.gt.u64 %p17, %rd13, %rd11;
; SM70-NEXT:    selp.b64 %rd34, %rd11, %rd33, %p17;
; SM70-NEXT:    selp.b64 %rd35, %rd11, %rd34, %p16;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd47, [%rd17+40], %rd13, %rd35;
; SM70-NEXT:    setp.ne.b64 %p18, %rd47, %rd13;
; SM70-NEXT:    @%p18 bra $L__BB207_11;
; SM70-NEXT:  // %bb.12: // %atomicrmw.end55
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd48, [%rd17+48];
; SM70-NEXT:  $L__BB207_13: // %atomicrmw.start68
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd15, %rd48;
; SM70-NEXT:    add.s64 %rd36, %rd15, -1;
; SM70-NEXT:    setp.eq.b64 %p19, %rd15, 0;
; SM70-NEXT:    setp.gt.u64 %p20, %rd15, %rd10;
; SM70-NEXT:    selp.b64 %rd37, %rd10, %rd36, %p20;
; SM70-NEXT:    selp.b64 %rd38, %rd10, %rd37, %p19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd48, [%rd17+48], %rd15, %rd38;
; SM70-NEXT:    setp.ne.b64 %p21, %rd48, %rd15;
; SM70-NEXT:    @%p21 bra $L__BB207_13;
; SM70-NEXT:  // %bb.14: // %atomicrmw.end67
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd49, [%rd17+56];
; SM70-NEXT:  $L__BB207_15: // %atomicrmw.start76
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    add.s64 %rd39, %rd49, -1;
; SM70-NEXT:    setp.eq.b64 %p22, %rd49, 0;
; SM70-NEXT:    setp.gt.u64 %p23, %rd49, %rd14;
; SM70-NEXT:    selp.b64 %rd40, %rd14, %rd39, %p23;
; SM70-NEXT:    selp.b64 %rd41, %rd14, %rd40, %p22;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd49, %rd41;
; SM70-NEXT:    setp.ne.b64 %p24, %rd16, %rd49;
; SM70-NEXT:    mov.b64 %rd49, %rd16;
; SM70-NEXT:    @%p24 bra $L__BB207_15;
; SM70-NEXT:  // %bb.16: // %atomicrmw.end75
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd48, %rd16};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd46, %rd47};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd42, %rd43};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd44, %rd45};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise udec_wrap ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @usub_cond_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: usub_cond_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b16 %rs<6>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b8 %rs1, [usub_cond_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd2, [usub_cond_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 255, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:  $L__BB208_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r14, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    and.b16 %rs3, %rs2, 255;
; SM70-NEXT:    setp.ge.u16 %p1, %rs3, %rs1;
; SM70-NEXT:    sub.s16 %rs4, %rs2, %rs1;
; SM70-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs5;
; SM70-NEXT:    and.b32 %r9, %r8, 255;
; SM70-NEXT:    shl.b32 %r10, %r9, %r1;
; SM70-NEXT:    and.b32 %r11, %r14, %r2;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p2, %r3, %r14;
; SM70-NEXT:    mov.b32 %r14, %r3;
; SM70-NEXT:    @%p2 bra $L__BB208_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @usub_cond_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: usub_cond_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<4>;
; SM70-NEXT:    .reg .b16 %rs<15>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [usub_cond_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT:    ld.param.b64 %rd2, [usub_cond_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    and.b32 %r6, %r5, 3;
; SM70-NEXT:    shl.b32 %r1, %r6, 3;
; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r7;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:    mov.b32 {%rs6, %rs7}, %r4;
; SM70-NEXT:  $L__BB209_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r8, %r14, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r8;
; SM70-NEXT:    and.b16 %rs4, %rs3, 255;
; SM70-NEXT:    shr.u16 %rs5, %rs3, 8;
; SM70-NEXT:    setp.ge.u16 %p1, %rs5, %rs7;
; SM70-NEXT:    setp.ge.u16 %p2, %rs4, %rs6;
; SM70-NEXT:    sub.s16 %rs8, %rs5, %rs7;
; SM70-NEXT:    sub.s16 %rs9, %rs3, %rs6;
; SM70-NEXT:    selp.b16 %rs10, %rs9, %rs3, %p2;
; SM70-NEXT:    selp.b16 %rs11, %rs8, %rs5, %p1;
; SM70-NEXT:    shl.b16 %rs12, %rs11, 8;
; SM70-NEXT:    and.b16 %rs13, %rs10, 255;
; SM70-NEXT:    or.b16 %rs14, %rs13, %rs12;
; SM70-NEXT:    cvt.u32.u16 %r9, %rs14;
; SM70-NEXT:    shl.b32 %r10, %r9, %r1;
; SM70-NEXT:    and.b32 %r11, %r14, %r2;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p3, %r3, %r14;
; SM70-NEXT:    mov.b32 %r14, %r3;
; SM70-NEXT:    @%p3 bra $L__BB209_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @usub_cond_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: usub_cond_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<6>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<23>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [usub_cond_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [usub_cond_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r22, [%rd1];
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r5;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r3;
; SM70-NEXT:  $L__BB210_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r4, %r22, 0, 0x7770U;
; SM70-NEXT:    setp.ge.u32 %p1, %r4, %r3;
; SM70-NEXT:    prmt.b32 %r6, %r22, 0, 0x7771U;
; SM70-NEXT:    setp.ge.u32 %p2, %r6, %r5;
; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
; SM70-NEXT:    prmt.b32 %r8, %r22, 0, 0x7772U;
; SM70-NEXT:    setp.ge.u32 %p3, %r8, %r7;
; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0x7773U;
; SM70-NEXT:    prmt.b32 %r10, %r22, 0, 0x7773U;
; SM70-NEXT:    setp.ge.u32 %p4, %r10, %r9;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r9;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r4;
; SM70-NEXT:    sub.s16 %rs6, %rs5, %rs4;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r6;
; SM70-NEXT:    sub.s16 %rs8, %rs7, %rs3;
; SM70-NEXT:    cvt.u16.u32 %rs9, %r8;
; SM70-NEXT:    sub.s16 %rs10, %rs9, %rs2;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r10;
; SM70-NEXT:    sub.s16 %rs12, %rs11, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r11, %rs12;
; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p4;
; SM70-NEXT:    cvt.u32.u16 %r13, %rs10;
; SM70-NEXT:    selp.b32 %r14, %r13, %r8, %p3;
; SM70-NEXT:    prmt.b32 %r15, %r14, %r12, 0x3340U;
; SM70-NEXT:    cvt.u32.u16 %r16, %rs8;
; SM70-NEXT:    selp.b32 %r17, %r16, %r6, %p2;
; SM70-NEXT:    cvt.u32.u16 %r18, %rs6;
; SM70-NEXT:    selp.b32 %r19, %r18, %r4, %p1;
; SM70-NEXT:    prmt.b32 %r20, %r19, %r17, 0x3340U;
; SM70-NEXT:    prmt.b32 %r21, %r20, %r15, 0x5410U;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r22, %r21;
; SM70-NEXT:    setp.ne.b32 %p5, %r1, %r22;
; SM70-NEXT:    mov.b32 %r22, %r1;
; SM70-NEXT:    @%p5 bra $L__BB210_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @usub_cond_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: usub_cond_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<10>;
; SM70-NEXT:    .reg .b16 %rs<33>;
; SM70-NEXT:    .reg .b32 %r<35>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [usub_cond_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NEXT:  $L__BB211_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r33;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r34;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7770U;
; SM70-NEXT:    prmt.b32 %r4, %r34, 0, 0x7770U;
; SM70-NEXT:    setp.ge.u32 %p1, %r4, %r3;
; SM70-NEXT:    prmt.b32 %r5, %r2, 0, 0x7771U;
; SM70-NEXT:    prmt.b32 %r6, %r34, 0, 0x7771U;
; SM70-NEXT:    setp.ge.u32 %p2, %r6, %r5;
; SM70-NEXT:    prmt.b32 %r7, %r2, 0, 0x7772U;
; SM70-NEXT:    prmt.b32 %r8, %r34, 0, 0x7772U;
; SM70-NEXT:    setp.ge.u32 %p3, %r8, %r7;
; SM70-NEXT:    prmt.b32 %r9, %r2, 0, 0x7773U;
; SM70-NEXT:    prmt.b32 %r10, %r34, 0, 0x7773U;
; SM70-NEXT:    setp.ge.u32 %p4, %r10, %r9;
; SM70-NEXT:    prmt.b32 %r11, %r1, 0, 0x7770U;
; SM70-NEXT:    prmt.b32 %r12, %r33, 0, 0x7770U;
; SM70-NEXT:    setp.ge.u32 %p5, %r12, %r11;
; SM70-NEXT:    prmt.b32 %r13, %r1, 0, 0x7771U;
; SM70-NEXT:    prmt.b32 %r14, %r33, 0, 0x7771U;
; SM70-NEXT:    setp.ge.u32 %p6, %r14, %r13;
; SM70-NEXT:    prmt.b32 %r15, %r1, 0, 0x7772U;
; SM70-NEXT:    prmt.b32 %r16, %r33, 0, 0x7772U;
; SM70-NEXT:    setp.ge.u32 %p7, %r16, %r15;
; SM70-NEXT:    prmt.b32 %r17, %r1, 0, 0x7773U;
; SM70-NEXT:    prmt.b32 %r18, %r33, 0, 0x7773U;
; SM70-NEXT:    setp.ge.u32 %p8, %r18, %r17;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r17;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r18;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r15;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r16;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r13;
; SM70-NEXT:    cvt.u16.u32 %rs6, %r14;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r11;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r12;
; SM70-NEXT:    cvt.u16.u32 %rs9, %r9;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r10;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r7;
; SM70-NEXT:    cvt.u16.u32 %rs12, %r8;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r5;
; SM70-NEXT:    cvt.u16.u32 %rs14, %r6;
; SM70-NEXT:    cvt.u16.u32 %rs15, %r3;
; SM70-NEXT:    cvt.u16.u32 %rs16, %r4;
; SM70-NEXT:    sub.s16 %rs17, %rs16, %rs15;
; SM70-NEXT:    sub.s16 %rs18, %rs14, %rs13;
; SM70-NEXT:    sub.s16 %rs19, %rs12, %rs11;
; SM70-NEXT:    sub.s16 %rs20, %rs10, %rs9;
; SM70-NEXT:    sub.s16 %rs21, %rs8, %rs7;
; SM70-NEXT:    sub.s16 %rs22, %rs6, %rs5;
; SM70-NEXT:    sub.s16 %rs23, %rs4, %rs3;
; SM70-NEXT:    sub.s16 %rs24, %rs2, %rs1;
; SM70-NEXT:    selp.b16 %rs25, %rs24, %rs2, %p8;
; SM70-NEXT:    cvt.u32.u16 %r19, %rs25;
; SM70-NEXT:    selp.b16 %rs26, %rs23, %rs4, %p7;
; SM70-NEXT:    cvt.u32.u16 %r20, %rs26;
; SM70-NEXT:    prmt.b32 %r21, %r20, %r19, 0x3340U;
; SM70-NEXT:    selp.b16 %rs27, %rs22, %rs6, %p6;
; SM70-NEXT:    cvt.u32.u16 %r22, %rs27;
; SM70-NEXT:    selp.b16 %rs28, %rs21, %rs8, %p5;
; SM70-NEXT:    cvt.u32.u16 %r23, %rs28;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r22, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r24, %r21, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r25;
; SM70-NEXT:    selp.b16 %rs29, %rs20, %rs10, %p4;
; SM70-NEXT:    cvt.u32.u16 %r26, %rs29;
; SM70-NEXT:    selp.b16 %rs30, %rs19, %rs12, %p3;
; SM70-NEXT:    cvt.u32.u16 %r27, %rs30;
; SM70-NEXT:    prmt.b32 %r28, %r27, %r26, 0x3340U;
; SM70-NEXT:    selp.b16 %rs31, %rs18, %rs14, %p2;
; SM70-NEXT:    cvt.u32.u16 %r29, %rs31;
; SM70-NEXT:    selp.b16 %rs32, %rs17, %rs16, %p1;
; SM70-NEXT:    cvt.u32.u16 %r30, %rs32;
; SM70-NEXT:    prmt.b32 %r31, %r30, %r29, 0x3340U;
; SM70-NEXT:    prmt.b32 %r32, %r31, %r28, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r32;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p9, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NEXT:    @%p9 bra $L__BB211_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @usub_cond_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: usub_cond_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b16 %rs<5>;
; SM70-NEXT:    .reg .b32 %r<14>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [usub_cond_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %rs1, [usub_cond_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NEXT:  $L__BB212_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    setp.ge.u16 %p1, %rs2, %rs1;
; SM70-NEXT:    sub.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    selp.b16 %rs4, %rs3, %rs2, %p1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs4;
; SM70-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NEXT:    setp.ne.b32 %p2, %r3, %r13;
; SM70-NEXT:    mov.b32 %r13, %r3;
; SM70-NEXT:    @%p2 bra $L__BB212_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @usub_cond_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: usub_cond_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<4>;
; SM70-NEXT:    .reg .b16 %rs<9>;
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [usub_cond_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [usub_cond_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT:  $L__BB213_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-NEXT:    setp.ge.u16 %p1, %rs3, %rs1;
; SM70-NEXT:    setp.ge.u16 %p2, %rs4, %rs2;
; SM70-NEXT:    sub.s16 %rs5, %rs3, %rs1;
; SM70-NEXT:    sub.s16 %rs6, %rs4, %rs2;
; SM70-NEXT:    selp.b16 %rs7, %rs6, %rs4, %p2;
; SM70-NEXT:    selp.b16 %rs8, %rs5, %rs3, %p1;
; SM70-NEXT:    mov.b32 %r3, {%rs8, %rs7};
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NEXT:    setp.ne.b32 %p3, %r1, %r4;
; SM70-NEXT:    mov.b32 %r4, %r1;
; SM70-NEXT:    @%p3 bra $L__BB213_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @usub_cond_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: usub_cond_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<6>;
; SM70-NEXT:    .reg .b16 %rs<17>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [usub_cond_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
; SM70-NEXT:  $L__BB214_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r6;
; SM70-NEXT:    setp.ge.u16 %p1, %rs3, %rs1;
; SM70-NEXT:    setp.ge.u16 %p2, %rs4, %rs2;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r5;
; SM70-NEXT:    setp.ge.u16 %p3, %rs7, %rs5;
; SM70-NEXT:    setp.ge.u16 %p4, %rs8, %rs6;
; SM70-NEXT:    sub.s16 %rs9, %rs3, %rs1;
; SM70-NEXT:    sub.s16 %rs10, %rs4, %rs2;
; SM70-NEXT:    sub.s16 %rs11, %rs7, %rs5;
; SM70-NEXT:    sub.s16 %rs12, %rs8, %rs6;
; SM70-NEXT:    selp.b16 %rs13, %rs12, %rs8, %p4;
; SM70-NEXT:    selp.b16 %rs14, %rs11, %rs7, %p3;
; SM70-NEXT:    mov.b32 %r3, {%rs14, %rs13};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r3;
; SM70-NEXT:    selp.b16 %rs15, %rs10, %rs4, %p2;
; SM70-NEXT:    selp.b16 %rs16, %rs9, %rs3, %p1;
; SM70-NEXT:    mov.b32 %r4, {%rs16, %rs15};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r4;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p5, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NEXT:    @%p5 bra $L__BB214_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @usub_cond_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: usub_cond_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<11>;
; SM70-NEXT:    .reg .b16 %rs<33>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [usub_cond_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
; SM70-NEXT:  $L__BB215_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
; SM70-NEXT:    setp.ge.u16 %p1, %rs3, %rs1;
; SM70-NEXT:    setp.ge.u16 %p2, %rs4, %rs2;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r9;
; SM70-NEXT:    setp.ge.u16 %p3, %rs7, %rs5;
; SM70-NEXT:    setp.ge.u16 %p4, %rs8, %rs6;
; SM70-NEXT:    sub.s16 %rs9, %rs3, %rs1;
; SM70-NEXT:    sub.s16 %rs10, %rs4, %rs2;
; SM70-NEXT:    sub.s16 %rs11, %rs7, %rs5;
; SM70-NEXT:    sub.s16 %rs12, %rs8, %rs6;
; SM70-NEXT:    selp.b16 %rs13, %rs12, %rs8, %p4;
; SM70-NEXT:    selp.b16 %rs14, %rs11, %rs7, %p3;
; SM70-NEXT:    mov.b32 %r5, {%rs14, %rs13};
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    selp.b16 %rs15, %rs10, %rs4, %p2;
; SM70-NEXT:    selp.b16 %rs16, %rs9, %rs3, %p1;
; SM70-NEXT:    mov.b32 %r6, {%rs16, %rs15};
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p5, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NEXT:    @%p5 bra $L__BB215_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NEXT:  $L__BB215_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    mov.b32 {%rs17, %rs18}, %r4;
; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r12;
; SM70-NEXT:    setp.ge.u16 %p6, %rs19, %rs17;
; SM70-NEXT:    setp.ge.u16 %p7, %rs20, %rs18;
; SM70-NEXT:    mov.b32 {%rs21, %rs22}, %r3;
; SM70-NEXT:    mov.b32 {%rs23, %rs24}, %r11;
; SM70-NEXT:    setp.ge.u16 %p8, %rs23, %rs21;
; SM70-NEXT:    setp.ge.u16 %p9, %rs24, %rs22;
; SM70-NEXT:    sub.s16 %rs25, %rs19, %rs17;
; SM70-NEXT:    sub.s16 %rs26, %rs20, %rs18;
; SM70-NEXT:    sub.s16 %rs27, %rs23, %rs21;
; SM70-NEXT:    sub.s16 %rs28, %rs24, %rs22;
; SM70-NEXT:    selp.b16 %rs29, %rs28, %rs24, %p9;
; SM70-NEXT:    selp.b16 %rs30, %rs27, %rs23, %p8;
; SM70-NEXT:    mov.b32 %r7, {%rs30, %rs29};
; SM70-NEXT:    cvt.u64.u32 %rd17, %r7;
; SM70-NEXT:    selp.b16 %rs31, %rs26, %rs20, %p7;
; SM70-NEXT:    selp.b16 %rs32, %rs25, %rs19, %p6;
; SM70-NEXT:    mov.b32 %r8, {%rs32, %rs31};
; SM70-NEXT:    cvt.u64.u32 %rd18, %r8;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
; SM70-NEXT:    setp.ne.b64 %p10, %rd21, %rd16;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NEXT:    @%p10 bra $L__BB215_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @usub_cond_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: usub_cond_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b32 %r<6>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [usub_cond_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [usub_cond_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT:  $L__BB216_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    setp.ge.u32 %p1, %r5, %r1;
; SM70-NEXT:    sub.s32 %r3, %r5, %r1;
; SM70-NEXT:    selp.b32 %r4, %r3, %r5, %p1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM70-NEXT:    setp.ne.b32 %p2, %r2, %r5;
; SM70-NEXT:    mov.b32 %r5, %r2;
; SM70-NEXT:    @%p2 bra $L__BB216_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @usub_cond_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: usub_cond_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<4>;
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [usub_cond_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [usub_cond_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:  $L__BB217_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r8;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    setp.ge.u32 %p1, %r8, %r2;
; SM70-NEXT:    setp.ge.u32 %p2, %r7, %r1;
; SM70-NEXT:    sub.s32 %r3, %r8, %r2;
; SM70-NEXT:    sub.s32 %r4, %r7, %r1;
; SM70-NEXT:    selp.b32 %r5, %r4, %r7, %p2;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    selp.b32 %r6, %r3, %r8, %p1;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p3, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd11;
; SM70-NEXT:    @%p3 bra $L__BB217_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @usub_cond_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: usub_cond_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<7>;
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [usub_cond_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r13, %rd2;
; SM70-NEXT:  $L__BB218_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r13;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r14;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    setp.ge.u32 %p1, %r14, %r2;
; SM70-NEXT:    setp.ge.u32 %p2, %r13, %r1;
; SM70-NEXT:    sub.s32 %r5, %r14, %r2;
; SM70-NEXT:    sub.s32 %r6, %r13, %r1;
; SM70-NEXT:    selp.b32 %r7, %r6, %r13, %p2;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r7;
; SM70-NEXT:    selp.b32 %r8, %r5, %r14, %p1;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r8;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p3, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r13, %rd11;
; SM70-NEXT:    @%p3 bra $L__BB218_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r15, %rd12;
; SM70-NEXT:  $L__BB218_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd13, %r15;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r16;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    setp.ge.u32 %p4, %r16, %r4;
; SM70-NEXT:    setp.ge.u32 %p5, %r15, %r3;
; SM70-NEXT:    sub.s32 %r9, %r16, %r4;
; SM70-NEXT:    sub.s32 %r10, %r15, %r3;
; SM70-NEXT:    selp.b32 %r11, %r10, %r15, %p5;
; SM70-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NEXT:    selp.b32 %r12, %r9, %r16, %p4;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
; SM70-NEXT:    setp.ne.b64 %p6, %rd21, %rd16;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r15, %rd21;
; SM70-NEXT:    @%p6 bra $L__BB218_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @usub_cond_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: usub_cond_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<13>;
; SM70-NEXT:    .reg .b32 %r<33>;
; SM70-NEXT:    .reg .b64 %rd<42>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [usub_cond_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_cond_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_cond_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r25, %rd2;
; SM70-NEXT:  $L__BB219_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r26;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    setp.ge.u32 %p1, %r26, %r6;
; SM70-NEXT:    setp.ge.u32 %p2, %r25, %r5;
; SM70-NEXT:    sub.s32 %r9, %r26, %r6;
; SM70-NEXT:    sub.s32 %r10, %r25, %r5;
; SM70-NEXT:    selp.b32 %r11, %r10, %r25, %p2;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r11;
; SM70-NEXT:    selp.b32 %r12, %r9, %r26, %p1;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r12;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p3, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r25, %rd11;
; SM70-NEXT:    @%p3 bra $L__BB219_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r27, %rd12;
; SM70-NEXT:  $L__BB219_3: // %atomicrmw.start6
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd13, %r27;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r28;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    setp.ge.u32 %p4, %r28, %r8;
; SM70-NEXT:    setp.ge.u32 %p5, %r27, %r7;
; SM70-NEXT:    sub.s32 %r13, %r28, %r8;
; SM70-NEXT:    sub.s32 %r14, %r27, %r7;
; SM70-NEXT:    selp.b32 %r15, %r14, %r27, %p5;
; SM70-NEXT:    cvt.u64.u32 %rd17, %r15;
; SM70-NEXT:    selp.b32 %r16, %r13, %r28, %p4;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r16;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd16, %rd20;
; SM70-NEXT:    setp.ne.b64 %p6, %rd21, %rd16;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r27, %rd21;
; SM70-NEXT:    @%p6 bra $L__BB219_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
; SM70-NEXT:    cvt.u32.u64 %r29, %rd22;
; SM70-NEXT:  $L__BB219_5: // %atomicrmw.start16
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd23, %r29;
; SM70-NEXT:    cvt.u64.u32 %rd24, %r30;
; SM70-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-NEXT:    setp.ge.u32 %p7, %r30, %r2;
; SM70-NEXT:    setp.ge.u32 %p8, %r29, %r1;
; SM70-NEXT:    sub.s32 %r17, %r30, %r2;
; SM70-NEXT:    sub.s32 %r18, %r29, %r1;
; SM70-NEXT:    selp.b32 %r19, %r18, %r29, %p8;
; SM70-NEXT:    cvt.u64.u32 %rd27, %r19;
; SM70-NEXT:    selp.b32 %r20, %r17, %r30, %p7;
; SM70-NEXT:    cvt.u64.u32 %rd28, %r20;
; SM70-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd26, %rd30;
; SM70-NEXT:    setp.ne.b64 %p9, %rd31, %rd26;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
; SM70-NEXT:    cvt.u32.u64 %r29, %rd31;
; SM70-NEXT:    @%p9 bra $L__BB219_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end15
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
; SM70-NEXT:    cvt.u32.u64 %r31, %rd32;
; SM70-NEXT:  $L__BB219_7: // %atomicrmw.start22
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd33, %r31;
; SM70-NEXT:    cvt.u64.u32 %rd34, %r32;
; SM70-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-NEXT:    setp.ge.u32 %p10, %r32, %r4;
; SM70-NEXT:    setp.ge.u32 %p11, %r31, %r3;
; SM70-NEXT:    sub.s32 %r21, %r32, %r4;
; SM70-NEXT:    sub.s32 %r22, %r31, %r3;
; SM70-NEXT:    selp.b32 %r23, %r22, %r31, %p11;
; SM70-NEXT:    cvt.u64.u32 %rd37, %r23;
; SM70-NEXT:    selp.b32 %r24, %r21, %r32, %p10;
; SM70-NEXT:    cvt.u64.u32 %rd38, %r24;
; SM70-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd36, %rd40;
; SM70-NEXT:    setp.ne.b64 %p12, %rd41, %rd36;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
; SM70-NEXT:    cvt.u32.u64 %r31, %rd41;
; SM70-NEXT:    @%p12 bra $L__BB219_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end21
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @usub_cond_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: usub_cond_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b64 %rd<7>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd3, [usub_cond_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd1, [usub_cond_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd6, [%rd3];
; SM70-NEXT:  $L__BB220_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    setp.ge.u64 %p1, %rd6, %rd1;
; SM70-NEXT:    sub.s64 %rd4, %rd6, %rd1;
; SM70-NEXT:    selp.b64 %rd5, %rd4, %rd6, %p1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
; SM70-NEXT:    setp.ne.b64 %p2, %rd2, %rd6;
; SM70-NEXT:    mov.b64 %rd6, %rd2;
; SM70-NEXT:    @%p2 bra $L__BB220_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @usub_cond_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: usub_cond_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<5>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd5, [usub_cond_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd1}, [usub_cond_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd10, [%rd5];
; SM70-NEXT:  $L__BB221_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd3, %rd10;
; SM70-NEXT:    setp.ge.u64 %p1, %rd3, %rd2;
; SM70-NEXT:    sub.s64 %rd6, %rd3, %rd2;
; SM70-NEXT:    selp.b64 %rd7, %rd6, %rd3, %p1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd10, [%rd5], %rd3, %rd7;
; SM70-NEXT:    setp.ne.b64 %p2, %rd10, %rd3;
; SM70-NEXT:    @%p2 bra $L__BB221_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
; SM70-NEXT:  $L__BB221_3: // %atomicrmw.start4
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    setp.ge.u64 %p3, %rd11, %rd1;
; SM70-NEXT:    sub.s64 %rd8, %rd11, %rd1;
; SM70-NEXT:    selp.b64 %rd9, %rd8, %rd11, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM70-NEXT:    setp.ne.b64 %p4, %rd4, %rd11;
; SM70-NEXT:    mov.b64 %rd11, %rd4;
; SM70-NEXT:    @%p4 bra $L__BB221_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end3
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @usub_cond_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: usub_cond_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<9>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd9, [usub_cond_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd6}, [usub_cond_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd3, %rd2}, [usub_cond_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd18, [%rd9];
; SM70-NEXT:  $L__BB222_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd4, %rd18;
; SM70-NEXT:    setp.ge.u64 %p1, %rd4, %rd3;
; SM70-NEXT:    sub.s64 %rd10, %rd4, %rd3;
; SM70-NEXT:    selp.b64 %rd11, %rd10, %rd4, %p1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd4, %rd11;
; SM70-NEXT:    setp.ne.b64 %p2, %rd18, %rd4;
; SM70-NEXT:    @%p2 bra $L__BB222_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
; SM70-NEXT:  $L__BB222_3: // %atomicrmw.start8
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd19;
; SM70-NEXT:    setp.ge.u64 %p3, %rd5, %rd2;
; SM70-NEXT:    sub.s64 %rd12, %rd5, %rd2;
; SM70-NEXT:    selp.b64 %rd13, %rd12, %rd5, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd5, %rd13;
; SM70-NEXT:    setp.ne.b64 %p4, %rd19, %rd5;
; SM70-NEXT:    @%p4 bra $L__BB222_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end7
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
; SM70-NEXT:  $L__BB222_5: // %atomicrmw.start20
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd7, %rd20;
; SM70-NEXT:    setp.ge.u64 %p5, %rd7, %rd1;
; SM70-NEXT:    sub.s64 %rd14, %rd7, %rd1;
; SM70-NEXT:    selp.b64 %rd15, %rd14, %rd7, %p5;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd20, [%rd9+16], %rd7, %rd15;
; SM70-NEXT:    setp.ne.b64 %p6, %rd20, %rd7;
; SM70-NEXT:    @%p6 bra $L__BB222_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end19
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
; SM70-NEXT:  $L__BB222_7: // %atomicrmw.start28
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    setp.ge.u64 %p7, %rd21, %rd6;
; SM70-NEXT:    sub.s64 %rd16, %rd21, %rd6;
; SM70-NEXT:    selp.b64 %rd17, %rd16, %rd21, %p7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM70-NEXT:    setp.ne.b64 %p8, %rd8, %rd21;
; SM70-NEXT:    mov.b64 %rd21, %rd8;
; SM70-NEXT:    @%p8 bra $L__BB222_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end27
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd20, %rd8};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @usub_cond_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: usub_cond_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<17>;
; SM70-NEXT:    .reg .b64 %rd<42>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd17, [usub_cond_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd10, %rd14}, [usub_cond_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd11}, [usub_cond_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd7}, [usub_cond_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd3}, [usub_cond_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd34, [%rd17];
; SM70-NEXT:  $L__BB223_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd34;
; SM70-NEXT:    setp.ge.u64 %p1, %rd5, %rd4;
; SM70-NEXT:    sub.s64 %rd18, %rd5, %rd4;
; SM70-NEXT:    selp.b64 %rd19, %rd18, %rd5, %p1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd5, %rd19;
; SM70-NEXT:    setp.ne.b64 %p2, %rd34, %rd5;
; SM70-NEXT:    @%p2 bra $L__BB223_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
; SM70-NEXT:  $L__BB223_3: // %atomicrmw.start12
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd6, %rd35;
; SM70-NEXT:    setp.ge.u64 %p3, %rd6, %rd3;
; SM70-NEXT:    sub.s64 %rd20, %rd6, %rd3;
; SM70-NEXT:    selp.b64 %rd21, %rd20, %rd6, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd6, %rd21;
; SM70-NEXT:    setp.ne.b64 %p4, %rd35, %rd6;
; SM70-NEXT:    @%p4 bra $L__BB223_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end11
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
; SM70-NEXT:  $L__BB223_5: // %atomicrmw.start24
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd8, %rd36;
; SM70-NEXT:    setp.ge.u64 %p5, %rd8, %rd2;
; SM70-NEXT:    sub.s64 %rd22, %rd8, %rd2;
; SM70-NEXT:    selp.b64 %rd23, %rd22, %rd8, %p5;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd8, %rd23;
; SM70-NEXT:    setp.ne.b64 %p6, %rd36, %rd8;
; SM70-NEXT:    @%p6 bra $L__BB223_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end23
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
; SM70-NEXT:  $L__BB223_7: // %atomicrmw.start32
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd9, %rd37;
; SM70-NEXT:    setp.ge.u64 %p7, %rd9, %rd7;
; SM70-NEXT:    sub.s64 %rd24, %rd9, %rd7;
; SM70-NEXT:    selp.b64 %rd25, %rd24, %rd9, %p7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd9, %rd25;
; SM70-NEXT:    setp.ne.b64 %p8, %rd37, %rd9;
; SM70-NEXT:    @%p8 bra $L__BB223_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end31
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
; SM70-NEXT:  $L__BB223_9: // %atomicrmw.start48
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd12, %rd38;
; SM70-NEXT:    setp.ge.u64 %p9, %rd12, %rd1;
; SM70-NEXT:    sub.s64 %rd26, %rd12, %rd1;
; SM70-NEXT:    selp.b64 %rd27, %rd26, %rd12, %p9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd12, %rd27;
; SM70-NEXT:    setp.ne.b64 %p10, %rd38, %rd12;
; SM70-NEXT:    @%p10 bra $L__BB223_9;
; SM70-NEXT:  // %bb.10: // %atomicrmw.end47
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
; SM70-NEXT:  $L__BB223_11: // %atomicrmw.start56
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd13, %rd39;
; SM70-NEXT:    setp.ge.u64 %p11, %rd13, %rd11;
; SM70-NEXT:    sub.s64 %rd28, %rd13, %rd11;
; SM70-NEXT:    selp.b64 %rd29, %rd28, %rd13, %p11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd13, %rd29;
; SM70-NEXT:    setp.ne.b64 %p12, %rd39, %rd13;
; SM70-NEXT:    @%p12 bra $L__BB223_11;
; SM70-NEXT:  // %bb.12: // %atomicrmw.end55
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
; SM70-NEXT:  $L__BB223_13: // %atomicrmw.start68
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd15, %rd40;
; SM70-NEXT:    setp.ge.u64 %p13, %rd15, %rd10;
; SM70-NEXT:    sub.s64 %rd30, %rd15, %rd10;
; SM70-NEXT:    selp.b64 %rd31, %rd30, %rd15, %p13;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd40, [%rd17+48], %rd15, %rd31;
; SM70-NEXT:    setp.ne.b64 %p14, %rd40, %rd15;
; SM70-NEXT:    @%p14 bra $L__BB223_13;
; SM70-NEXT:  // %bb.14: // %atomicrmw.end67
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
; SM70-NEXT:  $L__BB223_15: // %atomicrmw.start76
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    setp.ge.u64 %p15, %rd41, %rd14;
; SM70-NEXT:    sub.s64 %rd32, %rd41, %rd14;
; SM70-NEXT:    selp.b64 %rd33, %rd32, %rd41, %p15;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM70-NEXT:    setp.ne.b64 %p16, %rd16, %rd41;
; SM70-NEXT:    mov.b64 %rd41, %rd16;
; SM70-NEXT:    @%p16 bra $L__BB223_15;
; SM70-NEXT:  // %bb.16: // %atomicrmw.end75
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd40, %rd16};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd34, %rd35};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_cond ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x i8> @usub_sat_acq_rel_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: usub_sat_acq_rel_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<6>;
; SM70-NEXT:    .reg .b32 %r<14>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b8 %rs1, [usub_sat_acq_rel_v1i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd2, [usub_sat_acq_rel_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 255, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NEXT:  $L__BB224_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    and.b16 %rs3, %rs2, 255;
; SM70-NEXT:    max.u16 %rs4, %rs3, %rs1;
; SM70-NEXT:    sub.s16 %rs5, %rs4, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs5;
; SM70-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r13;
; SM70-NEXT:    mov.b32 %r13, %r3;
; SM70-NEXT:    @%p1 bra $L__BB224_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r12;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acq_rel
        ret <1 x i8> %retval
}

define <2 x i8> @usub_sat_acq_rel_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: usub_sat_acq_rel_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<4>;
; SM70-NEXT:    .reg .b16 %rs<17>;
; SM70-NEXT:    .reg .b32 %r<21>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [usub_sat_acq_rel_v2i8_global_cta_param_1];
; SM70-NEXT:    mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT:    ld.param.b64 %rd2, [usub_sat_acq_rel_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    and.b32 %r6, %r5, 3;
; SM70-NEXT:    shl.b32 %r1, %r6, 3;
; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r7;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r20, [%rd1];
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
; SM70-NEXT:  $L__BB225_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r8, %r20, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r8;
; SM70-NEXT:    shr.u16 %rs4, %rs3, 8;
; SM70-NEXT:    mov.b32 %r9, {%rs3, %rs4};
; SM70-NEXT:    and.b32 %r10, %r9, 16711935;
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r10;
; SM70-NEXT:    sub.s16 %rs9, %rs6, %rs8;
; SM70-NEXT:    setp.gt.u16 %p1, %rs9, %rs6;
; SM70-NEXT:    selp.b16 %rs10, -1, 0, %p1;
; SM70-NEXT:    sub.s16 %rs11, %rs5, %rs7;
; SM70-NEXT:    setp.gt.u16 %p2, %rs11, %rs5;
; SM70-NEXT:    selp.b16 %rs12, -1, 0, %p2;
; SM70-NEXT:    mov.b32 %r11, {%rs12, %rs10};
; SM70-NEXT:    xor.b32 %r12, %r11, -1;
; SM70-NEXT:    mov.b32 %r13, {%rs11, %rs9};
; SM70-NEXT:    and.b32 %r14, %r13, %r12;
; SM70-NEXT:    mov.b32 {%rs13, %rs14}, %r14;
; SM70-NEXT:    shl.b16 %rs15, %rs14, 8;
; SM70-NEXT:    or.b16 %rs16, %rs13, %rs15;
; SM70-NEXT:    cvt.u32.u16 %r15, %rs16;
; SM70-NEXT:    shl.b32 %r16, %r15, %r1;
; SM70-NEXT:    and.b32 %r17, %r20, %r2;
; SM70-NEXT:    or.b32 %r18, %r17, %r16;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r20, %r18;
; SM70-NEXT:    setp.ne.b32 %p3, %r3, %r20;
; SM70-NEXT:    mov.b32 %r20, %r3;
; SM70-NEXT:    @%p3 bra $L__BB225_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r19, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r19;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acq_rel
        ret <2 x i8> %retval
}

define <4 x i8> @usub_sat_acq_rel_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: usub_sat_acq_rel_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<17>;
; SM70-NEXT:    .reg .b32 %r<19>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [usub_sat_acq_rel_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [usub_sat_acq_rel_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r6;
; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs9, %r10;
; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r13;
; SM70-NEXT:  $L__BB226_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    max.u16 %rs3, %rs2, %rs1;
; SM70-NEXT:    sub.s16 %rs4, %rs3, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs6, %r7;
; SM70-NEXT:    max.u16 %rs7, %rs6, %rs5;
; SM70-NEXT:    sub.s16 %rs8, %rs7, %rs5;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs8;
; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
; SM70-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r11;
; SM70-NEXT:    max.u16 %rs11, %rs10, %rs9;
; SM70-NEXT:    sub.s16 %rs12, %rs11, %rs9;
; SM70-NEXT:    cvt.u32.u16 %r12, %rs12;
; SM70-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs14, %r14;
; SM70-NEXT:    max.u16 %rs15, %rs14, %rs13;
; SM70-NEXT:    sub.s16 %rs16, %rs15, %rs13;
; SM70-NEXT:    cvt.u32.u16 %r15, %rs16;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r18;
; SM70-NEXT:    mov.b32 %r18, %r1;
; SM70-NEXT:    @%p1 bra $L__BB226_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acq_rel
        ret <4 x i8> %retval
}

define <8 x i8> @usub_sat_acq_rel_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: usub_sat_acq_rel_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<33>;
; SM70-NEXT:    .reg .b32 %r<35>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [usub_sat_acq_rel_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NEXT:  $L__BB227_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r3, %r1, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    max.u16 %rs3, %rs2, %rs1;
; SM70-NEXT:    sub.s16 %rs4, %rs3, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NEXT:    prmt.b32 %r6, %r1, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r6;
; SM70-NEXT:    prmt.b32 %r7, %r33, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs6, %r7;
; SM70-NEXT:    max.u16 %rs7, %rs6, %rs5;
; SM70-NEXT:    sub.s16 %rs8, %rs7, %rs5;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs8;
; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
; SM70-NEXT:    prmt.b32 %r10, %r1, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs9, %r10;
; SM70-NEXT:    prmt.b32 %r11, %r33, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r11;
; SM70-NEXT:    max.u16 %rs11, %rs10, %rs9;
; SM70-NEXT:    sub.s16 %rs12, %rs11, %rs9;
; SM70-NEXT:    cvt.u32.u16 %r12, %rs12;
; SM70-NEXT:    prmt.b32 %r13, %r1, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r13;
; SM70-NEXT:    prmt.b32 %r14, %r33, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs14, %r14;
; SM70-NEXT:    max.u16 %rs15, %rs14, %rs13;
; SM70-NEXT:    sub.s16 %rs16, %rs15, %rs13;
; SM70-NEXT:    cvt.u32.u16 %r15, %rs16;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r17;
; SM70-NEXT:    prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs17, %r18;
; SM70-NEXT:    prmt.b32 %r19, %r34, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs18, %r19;
; SM70-NEXT:    max.u16 %rs19, %rs18, %rs17;
; SM70-NEXT:    sub.s16 %rs20, %rs19, %rs17;
; SM70-NEXT:    cvt.u32.u16 %r20, %rs20;
; SM70-NEXT:    prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs21, %r21;
; SM70-NEXT:    prmt.b32 %r22, %r34, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs22, %r22;
; SM70-NEXT:    max.u16 %rs23, %rs22, %rs21;
; SM70-NEXT:    sub.s16 %rs24, %rs23, %rs21;
; SM70-NEXT:    cvt.u32.u16 %r23, %rs24;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r20, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs25, %r25;
; SM70-NEXT:    prmt.b32 %r26, %r34, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs26, %r26;
; SM70-NEXT:    max.u16 %rs27, %rs26, %rs25;
; SM70-NEXT:    sub.s16 %rs28, %rs27, %rs25;
; SM70-NEXT:    cvt.u32.u16 %r27, %rs28;
; SM70-NEXT:    prmt.b32 %r28, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs29, %r28;
; SM70-NEXT:    prmt.b32 %r29, %r34, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs30, %r29;
; SM70-NEXT:    max.u16 %rs31, %rs30, %rs29;
; SM70-NEXT:    sub.s16 %rs32, %rs31, %rs29;
; SM70-NEXT:    cvt.u32.u16 %r30, %rs32;
; SM70-NEXT:    prmt.b32 %r31, %r30, %r27, 0x3340U;
; SM70-NEXT:    prmt.b32 %r32, %r31, %r24, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r32;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB227_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acq_rel
        ret <8 x i8> %retval
}

define <1 x i16> @usub_sat_acq_rel_v1i16_global_cta(ptr addrspace(1) %addr, <1 x i16> %val) {
; SM70-LABEL: usub_sat_acq_rel_v1i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<5>;
; SM70-NEXT:    .reg .b32 %r<14>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [usub_sat_acq_rel_v1i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %rs1, [usub_sat_acq_rel_v1i16_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r6;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NEXT:  $L__BB228_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NEXT:    max.u16 %rs3, %rs2, %rs1;
; SM70-NEXT:    sub.s16 %rs4, %rs3, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs4;
; SM70-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r13;
; SM70-NEXT:    mov.b32 %r13, %r3;
; SM70-NEXT:    @%p1 bra $L__BB228_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <1 x i16> %val syncscope("block") acq_rel
        ret <1 x i16> %retval
}

define <2 x i16> @usub_sat_acq_rel_v2i16_global_cta(ptr addrspace(1) %addr, <2 x i16> %val) {
; SM70-LABEL: usub_sat_acq_rel_v2i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<4>;
; SM70-NEXT:    .reg .b16 %rs<9>;
; SM70-NEXT:    .reg .b32 %r<8>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [usub_sat_acq_rel_v2i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [usub_sat_acq_rel_v2i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NEXT:  $L__BB229_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r7;
; SM70-NEXT:    sub.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    sub.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    setp.gt.u16 %p1, %rs5, %rs4;
; SM70-NEXT:    selp.b16 %rs7, -1, 0, %p1;
; SM70-NEXT:    setp.gt.u16 %p2, %rs6, %rs3;
; SM70-NEXT:    selp.b16 %rs8, -1, 0, %p2;
; SM70-NEXT:    mov.b32 %r4, {%rs8, %rs7};
; SM70-NEXT:    xor.b32 %r5, %r4, -1;
; SM70-NEXT:    and.b32 %r6, %r3, %r5;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r7, %r6;
; SM70-NEXT:    setp.ne.b32 %p3, %r1, %r7;
; SM70-NEXT:    mov.b32 %r7, %r1;
; SM70-NEXT:    @%p3 bra $L__BB229_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <2 x i16> %val syncscope("block") acq_rel
        ret <2 x i16> %retval
}

define <4 x i16> @usub_sat_acq_rel_v4i16_global_cta(ptr addrspace(1) %addr, <4 x i16> %val) {
; SM70-LABEL: usub_sat_acq_rel_v4i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<6>;
; SM70-NEXT:    .reg .b16 %rs<17>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v4i16_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [usub_sat_acq_rel_v4i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
; SM70-NEXT:  $L__BB230_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r11;
; SM70-NEXT:    sub.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    sub.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r3, {%rs6, %rs5};
; SM70-NEXT:    setp.gt.u16 %p1, %rs5, %rs4;
; SM70-NEXT:    selp.b16 %rs7, -1, 0, %p1;
; SM70-NEXT:    setp.gt.u16 %p2, %rs6, %rs3;
; SM70-NEXT:    selp.b16 %rs8, -1, 0, %p2;
; SM70-NEXT:    mov.b32 %r4, {%rs8, %rs7};
; SM70-NEXT:    xor.b32 %r5, %r4, -1;
; SM70-NEXT:    and.b32 %r6, %r3, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r6;
; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r12;
; SM70-NEXT:    sub.s16 %rs13, %rs12, %rs10;
; SM70-NEXT:    sub.s16 %rs14, %rs11, %rs9;
; SM70-NEXT:    mov.b32 %r7, {%rs14, %rs13};
; SM70-NEXT:    setp.gt.u16 %p3, %rs13, %rs12;
; SM70-NEXT:    selp.b16 %rs15, -1, 0, %p3;
; SM70-NEXT:    setp.gt.u16 %p4, %rs14, %rs11;
; SM70-NEXT:    selp.b16 %rs16, -1, 0, %p4;
; SM70-NEXT:    mov.b32 %r8, {%rs16, %rs15};
; SM70-NEXT:    xor.b32 %r9, %r8, -1;
; SM70-NEXT:    and.b32 %r10, %r7, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r12;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd11;
; SM70-NEXT:    @%p5 bra $L__BB230_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <4 x i16> %val syncscope("block") acq_rel
        ret <4 x i16> %retval
}

define <8 x i16> @usub_sat_acq_rel_v8i16_global_cta(ptr addrspace(1) %addr, <8 x i16> %val) {
; SM70-LABEL: usub_sat_acq_rel_v8i16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<11>;
; SM70-NEXT:    .reg .b16 %rs<33>;
; SM70-NEXT:    .reg .b32 %r<25>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [usub_sat_acq_rel_v8i16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i16_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NEXT:    mov.b32 {%rs9, %rs10}, %r2;
; SM70-NEXT:  $L__BB231_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
; SM70-NEXT:    sub.s16 %rs5, %rs4, %rs2;
; SM70-NEXT:    sub.s16 %rs6, %rs3, %rs1;
; SM70-NEXT:    mov.b32 %r5, {%rs6, %rs5};
; SM70-NEXT:    setp.gt.u16 %p1, %rs5, %rs4;
; SM70-NEXT:    selp.b16 %rs7, -1, 0, %p1;
; SM70-NEXT:    setp.gt.u16 %p2, %rs6, %rs3;
; SM70-NEXT:    selp.b16 %rs8, -1, 0, %p2;
; SM70-NEXT:    mov.b32 %r6, {%rs8, %rs7};
; SM70-NEXT:    xor.b32 %r7, %r6, -1;
; SM70-NEXT:    and.b32 %r8, %r5, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r8;
; SM70-NEXT:    mov.b32 {%rs11, %rs12}, %r22;
; SM70-NEXT:    sub.s16 %rs13, %rs12, %rs10;
; SM70-NEXT:    sub.s16 %rs14, %rs11, %rs9;
; SM70-NEXT:    mov.b32 %r9, {%rs14, %rs13};
; SM70-NEXT:    setp.gt.u16 %p3, %rs13, %rs12;
; SM70-NEXT:    selp.b16 %rs15, -1, 0, %p3;
; SM70-NEXT:    setp.gt.u16 %p4, %rs14, %rs11;
; SM70-NEXT:    selp.b16 %rs16, -1, 0, %p4;
; SM70-NEXT:    mov.b32 %r10, {%rs16, %rs15};
; SM70-NEXT:    xor.b32 %r11, %r10, -1;
; SM70-NEXT:    and.b32 %r12, %r9, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r12;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-NEXT:    @%p5 bra $L__BB231_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r23, %rd12;
; SM70-NEXT:  $L__BB231_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b32 {%rs17, %rs18}, %r3;
; SM70-NEXT:    mov.b32 {%rs19, %rs20}, %r23;
; SM70-NEXT:    sub.s16 %rs21, %rs20, %rs18;
; SM70-NEXT:    sub.s16 %rs22, %rs19, %rs17;
; SM70-NEXT:    mov.b32 %r13, {%rs22, %rs21};
; SM70-NEXT:    setp.gt.u16 %p6, %rs21, %rs20;
; SM70-NEXT:    selp.b16 %rs23, -1, 0, %p6;
; SM70-NEXT:    setp.gt.u16 %p7, %rs22, %rs19;
; SM70-NEXT:    selp.b16 %rs24, -1, 0, %p7;
; SM70-NEXT:    mov.b32 %r14, {%rs24, %rs23};
; SM70-NEXT:    xor.b32 %r15, %r14, -1;
; SM70-NEXT:    and.b32 %r16, %r13, %r15;
; SM70-NEXT:    cvt.u64.u32 %rd13, %r16;
; SM70-NEXT:    mov.b32 {%rs25, %rs26}, %r4;
; SM70-NEXT:    mov.b32 {%rs27, %rs28}, %r24;
; SM70-NEXT:    sub.s16 %rs29, %rs28, %rs26;
; SM70-NEXT:    sub.s16 %rs30, %rs27, %rs25;
; SM70-NEXT:    mov.b32 %r17, {%rs30, %rs29};
; SM70-NEXT:    setp.gt.u16 %p8, %rs29, %rs28;
; SM70-NEXT:    selp.b16 %rs31, -1, 0, %p8;
; SM70-NEXT:    setp.gt.u16 %p9, %rs30, %rs27;
; SM70-NEXT:    selp.b16 %rs32, -1, 0, %p9;
; SM70-NEXT:    mov.b32 %r18, {%rs32, %rs31};
; SM70-NEXT:    xor.b32 %r19, %r18, -1;
; SM70-NEXT:    and.b32 %r20, %r17, %r19;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r20;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    cvt.u64.u32 %rd17, %r23;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r24;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r23, %rd21;
; SM70-NEXT:    @%p10 bra $L__BB231_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <8 x i16> %val syncscope("block") acq_rel
        ret <8 x i16> %retval
}

define <1 x i32> @usub_sat_acq_rel_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: usub_sat_acq_rel_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<6>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [usub_sat_acq_rel_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [usub_sat_acq_rel_v1i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT:  $L__BB232_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.u32 %r3, %r5, %r1;
; SM70-NEXT:    sub.s32 %r4, %r3, %r1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM70-NEXT:    setp.ne.b32 %p1, %r2, %r5;
; SM70-NEXT:    mov.b32 %r5, %r2;
; SM70-NEXT:    @%p1 bra $L__BB232_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acq_rel
        ret <1 x i32> %retval
}

define <2 x i32> @usub_sat_acq_rel_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: usub_sat_acq_rel_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [usub_sat_acq_rel_v2i32_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [usub_sat_acq_rel_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:  $L__BB233_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.u32 %r3, %r7, %r1;
; SM70-NEXT:    sub.s32 %r4, %r3, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r4;
; SM70-NEXT:    max.u32 %r5, %r8, %r2;
; SM70-NEXT:    sub.s32 %r6, %r5, %r2;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r8;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB233_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acq_rel
        ret <2 x i32> %retval
}

define <4 x i32> @usub_sat_acq_rel_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: usub_sat_acq_rel_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [usub_sat_acq_rel_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v4i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r13, %rd2;
; SM70-NEXT:  $L__BB234_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.u32 %r5, %r13, %r1;
; SM70-NEXT:    sub.s32 %r6, %r5, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r6;
; SM70-NEXT:    max.u32 %r7, %r14, %r2;
; SM70-NEXT:    sub.s32 %r8, %r7, %r2;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r8;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r13;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r14;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r14}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r13, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB234_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r15, %rd12;
; SM70-NEXT:  $L__BB234_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.u32 %r9, %r15, %r3;
; SM70-NEXT:    sub.s32 %r10, %r9, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd13, %r10;
; SM70-NEXT:    max.u32 %r11, %r16, %r4;
; SM70-NEXT:    sub.s32 %r12, %r11, %r4;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    cvt.u64.u32 %rd17, %r15;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r16;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r16}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r15, %rd21;
; SM70-NEXT:    @%p2 bra $L__BB234_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acq_rel
        ret <4 x i32> %retval
}

define <8 x i32> @usub_sat_acq_rel_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: usub_sat_acq_rel_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<5>;
; SM70-NEXT:    .reg .b32 %r<33>;
; SM70-NEXT:    .reg .b64 %rd<42>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [usub_sat_acq_rel_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [usub_sat_acq_rel_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [usub_sat_acq_rel_v8i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r25, %rd2;
; SM70-NEXT:  $L__BB235_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.u32 %r9, %r25, %r5;
; SM70-NEXT:    sub.s32 %r10, %r9, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r10;
; SM70-NEXT:    max.u32 %r11, %r26, %r6;
; SM70-NEXT:    sub.s32 %r12, %r11, %r6;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r12;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r25;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r26;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r26}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r25, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB235_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r27, %rd12;
; SM70-NEXT:  $L__BB235_3: // %atomicrmw.start6
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.u32 %r13, %r27, %r7;
; SM70-NEXT:    sub.s32 %r14, %r13, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd13, %r14;
; SM70-NEXT:    max.u32 %r15, %r28, %r8;
; SM70-NEXT:    sub.s32 %r16, %r15, %r8;
; SM70-NEXT:    cvt.u64.u32 %rd14, %r16;
; SM70-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NEXT:    cvt.u64.u32 %rd17, %r27;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r28;
; SM70-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r28}, %rd21; }
; SM70-NEXT:    cvt.u32.u64 %r27, %rd21;
; SM70-NEXT:    @%p2 bra $L__BB235_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd22; }
; SM70-NEXT:    cvt.u32.u64 %r29, %rd22;
; SM70-NEXT:  $L__BB235_5: // %atomicrmw.start16
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.u32 %r17, %r29, %r1;
; SM70-NEXT:    sub.s32 %r18, %r17, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd23, %r18;
; SM70-NEXT:    max.u32 %r19, %r30, %r2;
; SM70-NEXT:    sub.s32 %r20, %r19, %r2;
; SM70-NEXT:    cvt.u64.u32 %rd24, %r20;
; SM70-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-NEXT:    cvt.u64.u32 %rd27, %r29;
; SM70-NEXT:    cvt.u64.u32 %rd28, %r30;
; SM70-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-NEXT:    setp.ne.b64 %p3, %rd31, %rd30;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r30}, %rd31; }
; SM70-NEXT:    cvt.u32.u64 %r29, %rd31;
; SM70-NEXT:    @%p3 bra $L__BB235_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end15
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd32; }
; SM70-NEXT:    cvt.u32.u64 %r31, %rd32;
; SM70-NEXT:  $L__BB235_7: // %atomicrmw.start22
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.u32 %r21, %r31, %r3;
; SM70-NEXT:    sub.s32 %r22, %r21, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd33, %r22;
; SM70-NEXT:    max.u32 %r23, %r32, %r4;
; SM70-NEXT:    sub.s32 %r24, %r23, %r4;
; SM70-NEXT:    cvt.u64.u32 %rd34, %r24;
; SM70-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-NEXT:    cvt.u64.u32 %rd37, %r31;
; SM70-NEXT:    cvt.u64.u32 %rd38, %r32;
; SM70-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-NEXT:    setp.ne.b64 %p4, %rd41, %rd40;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r32}, %rd41; }
; SM70-NEXT:    cvt.u32.u64 %r31, %rd41;
; SM70-NEXT:    @%p4 bra $L__BB235_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end21
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r29, %r30, %r31, %r32};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r25, %r26, %r27, %r28};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acq_rel
        ret <8 x i32> %retval
}

define <1 x i64> @usub_sat_acq_rel_v1i64_global_cta(ptr addrspace(1) %addr, <1 x i64> %val) {
; SM70-LABEL: usub_sat_acq_rel_v1i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b64 %rd<7>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd3, [usub_sat_acq_rel_v1i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd1, [usub_sat_acq_rel_v1i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd6, [%rd3];
; SM70-NEXT:  $L__BB236_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.u64 %rd4, %rd6, %rd1;
; SM70-NEXT:    sub.s64 %rd5, %rd4, %rd1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd6, %rd5;
; SM70-NEXT:    setp.ne.b64 %p1, %rd2, %rd6;
; SM70-NEXT:    mov.b64 %rd6, %rd2;
; SM70-NEXT:    @%p1 bra $L__BB236_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <1 x i64> %val syncscope("block") acq_rel
        ret <1 x i64> %retval
}

define <2 x i64> @usub_sat_acq_rel_v2i64_global_cta(ptr addrspace(1) %addr, <2 x i64> %val) {
; SM70-LABEL: usub_sat_acq_rel_v2i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd5, [usub_sat_acq_rel_v2i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd1}, [usub_sat_acq_rel_v2i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd10, [%rd5];
; SM70-NEXT:  $L__BB237_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd3, %rd10;
; SM70-NEXT:    max.u64 %rd6, %rd3, %rd2;
; SM70-NEXT:    sub.s64 %rd7, %rd6, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd10, [%rd5], %rd3, %rd7;
; SM70-NEXT:    setp.ne.b64 %p1, %rd10, %rd3;
; SM70-NEXT:    @%p1 bra $L__BB237_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd11, [%rd5+8];
; SM70-NEXT:  $L__BB237_3: // %atomicrmw.start4
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.u64 %rd8, %rd11, %rd1;
; SM70-NEXT:    sub.s64 %rd9, %rd8, %rd1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd11, %rd9;
; SM70-NEXT:    setp.ne.b64 %p2, %rd4, %rd11;
; SM70-NEXT:    mov.b64 %rd11, %rd4;
; SM70-NEXT:    @%p2 bra $L__BB237_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end3
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <2 x i64> %val syncscope("block") acq_rel
        ret <2 x i64> %retval
}

define <4 x i64> @usub_sat_acq_rel_v4i64_global_cta(ptr addrspace(1) %addr, <4 x i64> %val) {
; SM70-LABEL: usub_sat_acq_rel_v4i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<5>;
; SM70-NEXT:    .reg .b64 %rd<22>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd9, [usub_sat_acq_rel_v4i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd6}, [usub_sat_acq_rel_v4i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd3, %rd2}, [usub_sat_acq_rel_v4i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd18, [%rd9];
; SM70-NEXT:  $L__BB238_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd4, %rd18;
; SM70-NEXT:    max.u64 %rd10, %rd4, %rd3;
; SM70-NEXT:    sub.s64 %rd11, %rd10, %rd3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd18, [%rd9], %rd4, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd18, %rd4;
; SM70-NEXT:    @%p1 bra $L__BB238_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd19, [%rd9+8];
; SM70-NEXT:  $L__BB238_3: // %atomicrmw.start8
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd19;
; SM70-NEXT:    max.u64 %rd12, %rd5, %rd2;
; SM70-NEXT:    sub.s64 %rd13, %rd12, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd19, [%rd9+8], %rd5, %rd13;
; SM70-NEXT:    setp.ne.b64 %p2, %rd19, %rd5;
; SM70-NEXT:    @%p2 bra $L__BB238_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end7
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd20, [%rd9+16];
; SM70-NEXT:  $L__BB238_5: // %atomicrmw.start20
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd7, %rd20;
; SM70-NEXT:    max.u64 %rd14, %rd7, %rd1;
; SM70-NEXT:    sub.s64 %rd15, %rd14, %rd1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd20, [%rd9+16], %rd7, %rd15;
; SM70-NEXT:    setp.ne.b64 %p3, %rd20, %rd7;
; SM70-NEXT:    @%p3 bra $L__BB238_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end19
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd21, [%rd9+24];
; SM70-NEXT:  $L__BB238_7: // %atomicrmw.start28
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.u64 %rd16, %rd21, %rd6;
; SM70-NEXT:    sub.s64 %rd17, %rd16, %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd21, %rd17;
; SM70-NEXT:    setp.ne.b64 %p4, %rd8, %rd21;
; SM70-NEXT:    mov.b64 %rd21, %rd8;
; SM70-NEXT:    @%p4 bra $L__BB238_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end27
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd20, %rd8};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd18, %rd19};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <4 x i64> %val syncscope("block") acq_rel
        ret <4 x i64> %retval
}

define <8 x i64> @usub_sat_acq_rel_v8i64_global_cta(ptr addrspace(1) %addr, <8 x i64> %val) {
; SM70-LABEL: usub_sat_acq_rel_v8i64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<9>;
; SM70-NEXT:    .reg .b64 %rd<42>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd17, [usub_sat_acq_rel_v8i64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd10, %rd14}, [usub_sat_acq_rel_v8i64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd11}, [usub_sat_acq_rel_v8i64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd7}, [usub_sat_acq_rel_v8i64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd3}, [usub_sat_acq_rel_v8i64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd34, [%rd17];
; SM70-NEXT:  $L__BB239_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd34;
; SM70-NEXT:    max.u64 %rd18, %rd5, %rd4;
; SM70-NEXT:    sub.s64 %rd19, %rd18, %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd34, [%rd17], %rd5, %rd19;
; SM70-NEXT:    setp.ne.b64 %p1, %rd34, %rd5;
; SM70-NEXT:    @%p1 bra $L__BB239_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd35, [%rd17+8];
; SM70-NEXT:  $L__BB239_3: // %atomicrmw.start12
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd6, %rd35;
; SM70-NEXT:    max.u64 %rd20, %rd6, %rd3;
; SM70-NEXT:    sub.s64 %rd21, %rd20, %rd3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd35, [%rd17+8], %rd6, %rd21;
; SM70-NEXT:    setp.ne.b64 %p2, %rd35, %rd6;
; SM70-NEXT:    @%p2 bra $L__BB239_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end11
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd36, [%rd17+16];
; SM70-NEXT:  $L__BB239_5: // %atomicrmw.start24
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd8, %rd36;
; SM70-NEXT:    max.u64 %rd22, %rd8, %rd2;
; SM70-NEXT:    sub.s64 %rd23, %rd22, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd36, [%rd17+16], %rd8, %rd23;
; SM70-NEXT:    setp.ne.b64 %p3, %rd36, %rd8;
; SM70-NEXT:    @%p3 bra $L__BB239_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end23
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd37, [%rd17+24];
; SM70-NEXT:  $L__BB239_7: // %atomicrmw.start32
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd9, %rd37;
; SM70-NEXT:    max.u64 %rd24, %rd9, %rd7;
; SM70-NEXT:    sub.s64 %rd25, %rd24, %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd37, [%rd17+24], %rd9, %rd25;
; SM70-NEXT:    setp.ne.b64 %p4, %rd37, %rd9;
; SM70-NEXT:    @%p4 bra $L__BB239_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end31
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd38, [%rd17+32];
; SM70-NEXT:  $L__BB239_9: // %atomicrmw.start48
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd12, %rd38;
; SM70-NEXT:    max.u64 %rd26, %rd12, %rd1;
; SM70-NEXT:    sub.s64 %rd27, %rd26, %rd1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd38, [%rd17+32], %rd12, %rd27;
; SM70-NEXT:    setp.ne.b64 %p5, %rd38, %rd12;
; SM70-NEXT:    @%p5 bra $L__BB239_9;
; SM70-NEXT:  // %bb.10: // %atomicrmw.end47
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd39, [%rd17+40];
; SM70-NEXT:  $L__BB239_11: // %atomicrmw.start56
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd13, %rd39;
; SM70-NEXT:    max.u64 %rd28, %rd13, %rd11;
; SM70-NEXT:    sub.s64 %rd29, %rd28, %rd11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd39, [%rd17+40], %rd13, %rd29;
; SM70-NEXT:    setp.ne.b64 %p6, %rd39, %rd13;
; SM70-NEXT:    @%p6 bra $L__BB239_11;
; SM70-NEXT:  // %bb.12: // %atomicrmw.end55
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd40, [%rd17+48];
; SM70-NEXT:  $L__BB239_13: // %atomicrmw.start68
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd15, %rd40;
; SM70-NEXT:    max.u64 %rd30, %rd15, %rd10;
; SM70-NEXT:    sub.s64 %rd31, %rd30, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd40, [%rd17+48], %rd15, %rd31;
; SM70-NEXT:    setp.ne.b64 %p7, %rd40, %rd15;
; SM70-NEXT:    @%p7 bra $L__BB239_13;
; SM70-NEXT:  // %bb.14: // %atomicrmw.end67
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd41, [%rd17+56];
; SM70-NEXT:  $L__BB239_15: // %atomicrmw.start76
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.u64 %rd32, %rd41, %rd14;
; SM70-NEXT:    sub.s64 %rd33, %rd32, %rd14;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd41, %rd33;
; SM70-NEXT:    setp.ne.b64 %p8, %rd16, %rd41;
; SM70-NEXT:    mov.b64 %rd41, %rd16;
; SM70-NEXT:    @%p8 bra $L__BB239_15;
; SM70-NEXT:  // %bb.16: // %atomicrmw.end75
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd40, %rd16};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd38, %rd39};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd34, %rd35};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd36, %rd37};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise usub_sat ptr  addrspace(1) %addr, <8 x i64> %val syncscope("block") acq_rel
        ret <8 x i64> %retval
}

define <1 x float> @fadd_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_v1f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB240_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r3, %r4, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r2, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r4, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB240_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_v1f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<3>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_v1f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<3>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_v1f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<3>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r1, [fadd_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r2, [%rd1], %r1;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
        ret <1 x float> %retval
}

define <2 x float> @fadd_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_v2f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB241_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r3, %r5, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r4, %r6, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB241_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_v2f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_v2f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_v2f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v2f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v2f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r1;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r4, [%rd1+4], %r2;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
        ret <2 x float> %retval
}

define <4 x float> @fadd_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_v4f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB242_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r5, %r9, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r6, %r10, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB242_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB242_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r7, %r11, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r8, %r12, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB242_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_v4f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<9>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_v4f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<9>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_v4f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<9>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v4f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v4f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r5, [%rd1], %r1;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r2;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r7, [%rd1+8], %r3;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r8, [%rd1+12], %r4;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
        ret <4 x float> %retval
}

define <8 x float> @fadd_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_v8f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v8f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB243_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r9, %r17, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r10, %r18, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB243_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r19, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB243_3: // %atomicrmw.start6
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r11, %r19, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r12, %r20, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r19, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB243_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd22;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB243_5: // %atomicrmw.start16
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r13, %r21, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd23, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r14, %r22, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd24, %r14;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd27, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd28, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p3, %rd31, %rd30;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd31;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p3 bra $L__BB243_5;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.6: // %atomicrmw.end15
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd32;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB243_7: // %atomicrmw.start22
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r15, %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd33, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r16, %r24, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd34, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd37, %r23;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd38, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p4, %rd41, %rd40;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd41;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p4 bra $L__BB243_7;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.8: // %atomicrmw.end21
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_v8f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v8f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_v8f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v8f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_v8f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v8f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f32_global_cta_param_1+16];
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fadd_acq_rel_v8f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r9, [%rd1], %r5;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r10, [%rd1+4], %r6;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r11, [%rd1+8], %r7;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r8;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r13, [%rd1+16], %r1;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r14, [%rd1+20], %r2;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r15, [%rd1+24], %r3;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r16, [%rd1+28], %r4;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
        ret <8 x float> %retval
}

define <1 x float> @fsub_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v1f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB244_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r3, %r4, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r2, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r4, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB244_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v1f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<4>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r2, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r2;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r3;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v1f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<4>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r2, %r1;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r2;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r3;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v1f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<4>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r1, [fsub_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r2, %r1;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r3, [%rd1], %r2;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r3;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
        ret <1 x float> %retval
}

define <2 x float> @fsub_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v2f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB245_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r3, %r5, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r4, %r6, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB245_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v2f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r4, [%rd1], %r3;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r5, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r5;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r6};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v2f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r4, [%rd1], %r3;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r5, %r2;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r5;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r6};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v2f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v2f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v2f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r3, %r1;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r4, [%rd1], %r3;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r5, %r2;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r6, [%rd1+4], %r5;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r6};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
        ret <2 x float> %retval
}

define <4 x float> @fsub_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB246_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r5, %r9, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r6, %r10, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB246_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB246_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r7, %r11, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r8, %r12, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB246_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r5, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r6, [%rd1], %r5;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r7, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r8, [%rd1+4], %r7;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r9, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r10, [%rd1+8], %r9;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r11, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r11;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r5, %r1;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r6, [%rd1], %r5;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r7, %r2;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r8, [%rd1+4], %r7;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r9, %r3;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r10, [%rd1+8], %r9;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r11, %r4;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r11;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v4f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v4f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v4f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r5, %r1;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r6, [%rd1], %r5;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r7, %r2;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r8, [%rd1+4], %r7;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r9, %r3;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r10, [%rd1+8], %r9;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r11, %r4;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r12, [%rd1+12], %r11;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r6, %r8, %r10, %r12};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
        ret <4 x float> %retval
}

define <8 x float> @fsub_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB247_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r9, %r17, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r10, %r18, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB247_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r19, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB247_3: // %atomicrmw.start6
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r11, %r19, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r12, %r20, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r19, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB247_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd22;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB247_5: // %atomicrmw.start16
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r13, %r21, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd23, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r14, %r22, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd24, %r14;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd27, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd28, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p3, %rd31, %rd30;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd31;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p3 bra $L__BB247_5;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.6: // %atomicrmw.end15
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd32;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB247_7: // %atomicrmw.start22
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r15, %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd33, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r16, %r24, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd34, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd37, %r23;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd38, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p4, %rd41, %rd40;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd41;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p4 bra $L__BB247_7;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.8: // %atomicrmw.end21
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r9, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r10, [%rd1], %r9;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r11, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r12, [%rd1+4], %r11;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r13, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r14, [%rd1+8], %r13;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r15, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r16, [%rd1+12], %r15;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r17, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r18, [%rd1+16], %r17;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r19, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r20, [%rd1+20], %r19;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r21, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r22, [%rd1+24], %r21;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f32 %r23, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r23;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r9, %r5;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r10, [%rd1], %r9;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r11, %r6;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r12, [%rd1+4], %r11;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r13, %r7;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r14, [%rd1+8], %r13;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r15, %r8;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r16, [%rd1+12], %r15;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r17, %r1;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r18, [%rd1+16], %r17;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r19, %r2;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r20, [%rd1+20], %r19;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r21, %r3;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r22, [%rd1+24], %r21;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f32 %r23, %r4;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r23;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v8f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v8f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f32_global_cta_param_1+16];
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fsub_acq_rel_v8f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r9, %r5;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r10, [%rd1], %r9;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r11, %r6;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r12, [%rd1+4], %r11;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r13, %r7;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r14, [%rd1+8], %r13;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r15, %r8;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r16, [%rd1+12], %r15;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r17, %r1;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r18, [%rd1+16], %r17;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r19, %r2;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r20, [%rd1+20], %r19;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r21, %r3;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r22, [%rd1+24], %r21;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f32 %r23, %r4;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.f32 %r24, [%rd1+28], %r23;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r18, %r20, %r22, %r24};
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r10, %r12, %r14, %r16};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
        ret <8 x float> %retval
}

define <1 x float> @fmin_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v1f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB248_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r3, %r4, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r2, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r4, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB248_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v1f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB248_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r3, %r4, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p1, %r2, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r4, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB248_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_v1f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:  $L__BB248_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r3, %r4, %r1;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r2, %r4;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r4, %r2;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB248_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_v1f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r1, [fmin_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-FTZ-ALLOW-NEXT:  $L__BB248_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r3, %r4, %r1;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p1, %r2, %r4;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r4, %r2;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB248_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
        ret <1 x float> %retval
}

define <2 x float> @fmin_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v2f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v2f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v2f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB249_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r3, %r5, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r4, %r6, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB249_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v2f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v2f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v2f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB249_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r3, %r5, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r4, %r6, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB249_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_v2f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v2f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v2f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB249_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r3, %r5, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r4, %r6, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB249_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_v2f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v2f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v2f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB249_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r3, %r5, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r4, %r6, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB249_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
        ret <2 x float> %retval
}

define <4 x float> @fmin_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v4f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v4f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v4f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB250_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r5, %r9, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r6, %r10, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB250_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB250_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r7, %r11, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r8, %r12, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB250_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v4f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v4f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v4f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB250_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r5, %r9, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r6, %r10, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB250_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB250_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r7, %r11, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r8, %r12, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p2 bra $L__BB250_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_v4f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v4f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v4f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB250_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r5, %r9, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r6, %r10, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB250_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB250_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r7, %r11, %r3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r8, %r12, %r4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p2 bra $L__BB250_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_v4f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v4f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v4f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB250_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r5, %r9, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r6, %r10, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB250_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-FTZ-ALLOW-NEXT:  $L__BB250_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r7, %r11, %r3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r8, %r12, %r4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p2 bra $L__BB250_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
        ret <4 x float> %retval
}

define <8 x float> @fmin_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v8f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v8f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmin_acq_rel_v8f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB251_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r9, %r17, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r10, %r18, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB251_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r19, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB251_3: // %atomicrmw.start6
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r11, %r19, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r12, %r20, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r19, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB251_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd22;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB251_5: // %atomicrmw.start15
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r13, %r21, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd23, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r14, %r22, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd24, %r14;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd27, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd28, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p3, %rd31, %rd30;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd31;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p3 bra $L__BB251_5;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd32;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB251_7: // %atomicrmw.start20
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r15, %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd33, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r16, %r24, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd34, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd37, %r23;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd38, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p4, %rd41, %rd40;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd41;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p4 bra $L__BB251_7;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v8f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<5>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v8f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmin_acq_rel_v8f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB251_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r9, %r17, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r10, %r18, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB251_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r19, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB251_3: // %atomicrmw.start6
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r11, %r19, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r12, %r20, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r19, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p2 bra $L__BB251_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd22;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB251_5: // %atomicrmw.start15
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r13, %r21, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd23, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r14, %r22, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd24, %r14;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd27, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd28, %r22;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p3, %rd31, %rd30;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd31;
; SM70-NOFTZ-ALLOW-NEXT:    @%p3 bra $L__BB251_5;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd32;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB251_7: // %atomicrmw.start20
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r15, %r23, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd33, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r16, %r24, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd34, %r16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd37, %r23;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd38, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p4, %rd41, %rd40;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd41;
; SM70-NOFTZ-ALLOW-NEXT:    @%p4 bra $L__BB251_7;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_v8f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<5>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v8f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f32_global_cta_param_1+16];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmin_acq_rel_v8f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB251_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r9, %r17, %r5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r10, %r18, %r6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB251_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r19, %rd12;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB251_3: // %atomicrmw.start6
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r11, %r19, %r7;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r12, %r20, %r8;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r19;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r20;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r19, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p2 bra $L__BB251_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd22;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB251_5: // %atomicrmw.start15
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r13, %r21, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd23, %r13;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r14, %r22, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd24, %r14;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd27, %r21;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd28, %r22;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p3, %rd31, %rd30;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd31;
; SM70-FTZ-DISALLOW-NEXT:    @%p3 bra $L__BB251_5;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd32;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB251_7: // %atomicrmw.start20
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r15, %r23, %r3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd33, %r15;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r16, %r24, %r4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd34, %r16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd37, %r23;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd38, %r24;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p4, %rd41, %rd40;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd41;
; SM70-FTZ-DISALLOW-NEXT:    @%p4 bra $L__BB251_7;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_v8f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<5>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v8f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f32_global_cta_param_1+16];
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmin_acq_rel_v8f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB251_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r9, %r17, %r5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r10, %r18, %r6;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB251_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r19, %rd12;
; SM70-FTZ-ALLOW-NEXT:  $L__BB251_3: // %atomicrmw.start6
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r11, %r19, %r7;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r12, %r20, %r8;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r19;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r20;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r19, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p2 bra $L__BB251_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd22;
; SM70-FTZ-ALLOW-NEXT:  $L__BB251_5: // %atomicrmw.start15
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r13, %r21, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd23, %r13;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r14, %r22, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd24, %r14;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd27, %r21;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd28, %r22;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p3, %rd31, %rd30;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd31;
; SM70-FTZ-ALLOW-NEXT:    @%p3 bra $L__BB251_5;
; SM70-FTZ-ALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd32;
; SM70-FTZ-ALLOW-NEXT:  $L__BB251_7: // %atomicrmw.start20
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r15, %r23, %r3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd33, %r15;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r16, %r24, %r4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd34, %r16;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd37, %r23;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd38, %r24;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p4, %rd41, %rd40;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd41;
; SM70-FTZ-ALLOW-NEXT:    @%p4 bra $L__BB251_7;
; SM70-FTZ-ALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
        ret <8 x float> %retval
}

define <1 x float> @fmax_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v1f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB252_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r3, %r4, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r2, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r4, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB252_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v1f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB252_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r3, %r4, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p1, %r2, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r4, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB252_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_v1f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:  $L__BB252_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r3, %r4, %r1;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r2, %r4;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r4, %r2;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB252_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_v1f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r1, [fmax_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-FTZ-ALLOW-NEXT:  $L__BB252_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r3, %r4, %r1;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r4, %r3;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p1, %r2, %r4;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r4, %r2;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB252_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
        ret <1 x float> %retval
}

define <2 x float> @fmax_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v2f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v2f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v2f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB253_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r3, %r5, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r4, %r6, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB253_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v2f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v2f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v2f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB253_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r3, %r5, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r4, %r6, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB253_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_v2f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v2f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v2f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB253_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r3, %r5, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r4, %r6, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB253_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_v2f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v2f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v2f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB253_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r3, %r5, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r4, %r6, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB253_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
        ret <2 x float> %retval
}

define <4 x float> @fmax_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v4f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v4f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v4f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB254_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r5, %r9, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r6, %r10, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB254_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB254_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r7, %r11, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r8, %r12, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB254_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v4f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v4f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v4f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB254_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r5, %r9, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r6, %r10, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB254_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB254_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r7, %r11, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r8, %r12, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p2 bra $L__BB254_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_v4f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v4f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v4f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB254_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r5, %r9, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r6, %r10, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB254_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB254_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r7, %r11, %r3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r8, %r12, %r4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p2 bra $L__BB254_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_v4f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v4f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v4f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB254_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r5, %r9, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r6, %r10, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB254_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-FTZ-ALLOW-NEXT:  $L__BB254_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r7, %r11, %r3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r8, %r12, %r4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p2 bra $L__BB254_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
        ret <4 x float> %retval
}

define <8 x float> @fmax_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v8f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v8f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmax_acq_rel_v8f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB255_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r9, %r17, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r10, %r18, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB255_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r19, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB255_3: // %atomicrmw.start6
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r11, %r19, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r12, %r20, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r19, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB255_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd22;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB255_5: // %atomicrmw.start15
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r13, %r21, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd23, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r14, %r22, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd24, %r14;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd27, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd28, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p3, %rd31, %rd30;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd31;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p3 bra $L__BB255_5;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd32;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB255_7: // %atomicrmw.start20
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r15, %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd33, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r16, %r24, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd34, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd37, %r23;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd38, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p4, %rd41, %rd40;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd41;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p4 bra $L__BB255_7;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v8f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<5>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v8f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmax_acq_rel_v8f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB255_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r9, %r17, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r10, %r18, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB255_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r19, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB255_3: // %atomicrmw.start6
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r11, %r19, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r12, %r20, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r19, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p2 bra $L__BB255_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd22;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB255_5: // %atomicrmw.start15
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r13, %r21, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd23, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r14, %r22, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd24, %r14;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd27, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd28, %r22;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p3, %rd31, %rd30;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd31;
; SM70-NOFTZ-ALLOW-NEXT:    @%p3 bra $L__BB255_5;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd32;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB255_7: // %atomicrmw.start20
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r15, %r23, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd33, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r16, %r24, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd34, %r16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd37, %r23;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd38, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p4, %rd41, %rd40;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd41;
; SM70-NOFTZ-ALLOW-NEXT:    @%p4 bra $L__BB255_7;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_v8f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<5>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v8f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f32_global_cta_param_1+16];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmax_acq_rel_v8f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB255_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r9, %r17, %r5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r10, %r18, %r6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB255_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r19, %rd12;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB255_3: // %atomicrmw.start6
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r11, %r19, %r7;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r12, %r20, %r8;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r19;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r20;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r19, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p2 bra $L__BB255_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd22;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB255_5: // %atomicrmw.start15
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r13, %r21, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd23, %r13;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r14, %r22, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd24, %r14;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd27, %r21;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd28, %r22;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p3, %rd31, %rd30;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd31;
; SM70-FTZ-DISALLOW-NEXT:    @%p3 bra $L__BB255_5;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd32;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB255_7: // %atomicrmw.start20
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r15, %r23, %r3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd33, %r15;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r16, %r24, %r4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd34, %r16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd37, %r23;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd38, %r24;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p4, %rd41, %rd40;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd41;
; SM70-FTZ-DISALLOW-NEXT:    @%p4 bra $L__BB255_7;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_v8f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<5>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v8f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f32_global_cta_param_1+16];
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmax_acq_rel_v8f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB255_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r9, %r17, %r5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r10, %r18, %r6;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB255_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r19, %rd12;
; SM70-FTZ-ALLOW-NEXT:  $L__BB255_3: // %atomicrmw.start6
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r11, %r19, %r7;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r11;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r12, %r20, %r8;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r12;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r19;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r20;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r19, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p2 bra $L__BB255_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd22; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd22;
; SM70-FTZ-ALLOW-NEXT:  $L__BB255_5: // %atomicrmw.start15
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r13, %r21, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd23, %r13;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r14, %r22, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd24, %r14;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd27, %r21;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd28, %r22;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p3, %rd31, %rd30;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd31; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd31;
; SM70-FTZ-ALLOW-NEXT:    @%p3 bra $L__BB255_5;
; SM70-FTZ-ALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd32; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd32;
; SM70-FTZ-ALLOW-NEXT:  $L__BB255_7: // %atomicrmw.start20
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r15, %r23, %r3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd33, %r15;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r16, %r24, %r4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd34, %r16;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd37, %r23;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd38, %r24;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p4, %rd41, %rd40;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd41; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd41;
; SM70-FTZ-ALLOW-NEXT:    @%p4 bra $L__BB255_7;
; SM70-FTZ-ALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
        ret <8 x float> %retval
}

define <1 x float> @fminimum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_v1f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<8>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB256_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r7, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r3, %r7, %r1, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r7, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r4, %r7, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p3, %r7, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p4, %r2, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r7, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p4 bra $L__BB256_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v1f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<5>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<8>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB256_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r7, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r3, %r7, %r1, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r7, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r4, %r7, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p3, %r7, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p4, %r2, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r7, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    @%p4 bra $L__BB256_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v1f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<5>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<8>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:  $L__BB256_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r7, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r3, %r7, %r1, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r7, %r1;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r4, %r7, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r7, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p4, %r2, %r7;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r7, %r2;
; SM70-FTZ-DISALLOW-NEXT:    @%p4 bra $L__BB256_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_v1f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<5>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<8>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r1, [fminimum_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-FTZ-ALLOW-NEXT:  $L__BB256_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r7, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r3, %r7, %r1, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r7, %r1;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r4, %r7, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r7, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p4, %r2, %r7;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r7, %r2;
; SM70-FTZ-ALLOW-NEXT:    @%p4 bra $L__BB256_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
        ret <1 x float> %retval
}

define <2 x float> @fminimum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_v2f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<8>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v2f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB257_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r11, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r3, %r11, %r1, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r11, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r4, %r11, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p3, %r11, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p4, %r12, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r7, %r12, %r2, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p5, %r12, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r8, %r12, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r9, 0f7FC00000, %r8, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p6, %r12, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r10, %r7, %r9, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p7 bra $L__BB257_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r11, %r12};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v2f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<8>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v2f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB257_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r11, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r3, %r11, %r1, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r11, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r4, %r11, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p3, %r11, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p4, %r12, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r7, %r12, %r2, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p5, %r12, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r8, %r12, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r9, 0f7FC00000, %r8, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p6, %r12, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r10, %r7, %r9, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p7 bra $L__BB257_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r11, %r12};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v2f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<8>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v2f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB257_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r11, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r3, %r11, %r1, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r11, %r1;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r4, %r11, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r11, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r6;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p4, %r12, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r7, %r12, %r2, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p5, %r12, %r2;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r8, %r12, %r2;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r9, 0f7FC00000, %r8, %p5;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p6, %r12, %r2;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r10, %r7, %r9, %p6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r11;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p7 bra $L__BB257_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r11, %r12};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_v2f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<8>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v2f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v2f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB257_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r11, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r3, %r11, %r1, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r11, %r1;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r4, %r11, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r11, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r6;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p4, %r12, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r7, %r12, %r2, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p5, %r12, %r2;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r8, %r12, %r2;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r9, 0f7FC00000, %r8, %p5;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p6, %r12, %r2;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r10, %r7, %r9, %p6;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r11;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r12;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p7 bra $L__BB257_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r11, %r12};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
        ret <2 x float> %retval
}

define <4 x float> @fminimum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_v4f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<15>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v4f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB258_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r21, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r5, %r21, %r1, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r21, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r6, %r21, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r7, 0f7FC00000, %r6, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p3, %r21, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r8, %r5, %r7, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p4, %r22, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r9, %r22, %r2, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p5, %r22, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r10, %r22, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p6, %r22, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p7 bra $L__BB258_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB258_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p8, %r23, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r13, %r23, %r3, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p9, %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r14, %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p10, %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p10;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p11, %r24, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r17, %r24, %r4, %p11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p12, %r24, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r18, %r24, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p13, %r24, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p13;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r23;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p14 bra $L__BB258_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v4f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<15>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v4f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB258_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r21, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r5, %r21, %r1, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r21, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r6, %r21, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r7, 0f7FC00000, %r6, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p3, %r21, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r8, %r5, %r7, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p4, %r22, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r9, %r22, %r2, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p5, %r22, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r10, %r22, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p6, %r22, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p7 bra $L__BB258_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB258_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p8, %r23, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r13, %r23, %r3, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p9, %r23, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r14, %r23, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p10, %r23, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p10;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p11, %r24, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r17, %r24, %r4, %p11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p12, %r24, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r18, %r24, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p13, %r24, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p13;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r23;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p14 bra $L__BB258_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v4f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<15>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v4f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB258_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r21, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r5, %r21, %r1, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r21, %r1;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r6, %r21, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r7, 0f7FC00000, %r6, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r21, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r8, %r5, %r7, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r8;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p4, %r22, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r9, %r22, %r2, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p5, %r22, %r2;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r10, %r22, %r2;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p5;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p6, %r22, %r2;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p7 bra $L__BB258_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd12;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB258_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p8, %r23, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r13, %r23, %r3, %p8;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r23, %r3;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r14, %r23, %r3;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p9;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p10, %r23, %r3;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p10;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r16;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p11, %r24, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r17, %r24, %r4, %p11;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p12, %r24, %r4;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r18, %r24, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p13, %r24, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p13;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r20;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r23;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r24;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p14 bra $L__BB258_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_v4f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<15>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v4f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v4f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB258_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r21, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r5, %r21, %r1, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r21, %r1;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r6, %r21, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r7, 0f7FC00000, %r6, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r21, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r8, %r5, %r7, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r8;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p4, %r22, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r9, %r22, %r2, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p5, %r22, %r2;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r10, %r22, %r2;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p5;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p6, %r22, %r2;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p6;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r12;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p7 bra $L__BB258_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd12;
; SM70-FTZ-ALLOW-NEXT:  $L__BB258_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p8, %r23, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r13, %r23, %r3, %p8;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r23, %r3;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r14, %r23, %r3;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p9;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p10, %r23, %r3;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p10;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r16;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p11, %r24, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r17, %r24, %r4, %p11;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p12, %r24, %r4;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r18, %r24, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p13, %r24, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p13;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r20;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r23;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r24;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p14 bra $L__BB258_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
        ret <4 x float> %retval
}

define <8 x float> @fminimum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_v8f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<29>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<49>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v8f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB259_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r41, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r9, %r41, %r5, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r41, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r10, %r41, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p3, %r41, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p4, %r42, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r13, %r42, %r6, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p5, %r42, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r14, %r42, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p6, %r42, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p7 bra $L__BB259_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB259_3: // %atomicrmw.start6
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p8, %r43, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r17, %r43, %r7, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p9, %r43, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r18, %r43, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p10, %r43, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p10;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p11, %r44, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r21, %r44, %r8, %p11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p12, %r44, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r22, %r44, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r23, 0f7FC00000, %r22, %p12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p13, %r44, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r24, %r21, %r23, %p13;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p14 bra $L__BB259_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd22; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd22;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB259_5: // %atomicrmw.start15
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p15, %r45, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r25, %r45, %r1, %p15;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p16, %r45, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r26, %r45, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r27, 0f7FC00000, %r26, %p16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p17, %r45, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r28, %r25, %r27, %p17;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd23, %r28;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p18, %r46, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r29, %r46, %r2, %p18;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p19, %r46, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r30, %r46, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r31, 0f7FC00000, %r30, %p19;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p20, %r46, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r32, %r29, %r31, %p20;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd24, %r32;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd27, %r45;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd28, %r46;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p21, %rd31, %rd30;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd31; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd31;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p21 bra $L__BB259_5;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd32; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r47, %rd32;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB259_7: // %atomicrmw.start20
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p22, %r47, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r33, %r47, %r3, %p22;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p23, %r47, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r34, %r47, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r35, 0f7FC00000, %r34, %p23;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p24, %r47, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r36, %r33, %r35, %p24;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd33, %r36;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p25, %r48, -2147483648;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r37, %r48, %r4, %p25;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p26, %r48, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r38, %r48, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r39, 0f7FC00000, %r38, %p26;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p27, %r48, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r40, %r37, %r39, %p27;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd34, %r40;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd37, %r47;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd38, %r48;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p28, %rd41, %rd40;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd41; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r47, %rd41;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p28 bra $L__BB259_7;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r45, %r46, %r47, %r48};
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v8f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<29>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<49>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v8f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB259_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r41, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r9, %r41, %r5, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r41, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r10, %r41, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p3, %r41, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p4, %r42, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r13, %r42, %r6, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p5, %r42, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r14, %r42, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p6, %r42, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p7 bra $L__BB259_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB259_3: // %atomicrmw.start6
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p8, %r43, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r17, %r43, %r7, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p9, %r43, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r18, %r43, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p10, %r43, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p10;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p11, %r44, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r21, %r44, %r8, %p11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p12, %r44, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r22, %r44, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r23, 0f7FC00000, %r22, %p12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p13, %r44, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r24, %r21, %r23, %p13;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p14 bra $L__BB259_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd22; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd22;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB259_5: // %atomicrmw.start15
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p15, %r45, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r25, %r45, %r1, %p15;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p16, %r45, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r26, %r45, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r27, 0f7FC00000, %r26, %p16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p17, %r45, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r28, %r25, %r27, %p17;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd23, %r28;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p18, %r46, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r29, %r46, %r2, %p18;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p19, %r46, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r30, %r46, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r31, 0f7FC00000, %r30, %p19;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p20, %r46, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r32, %r29, %r31, %p20;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd24, %r32;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd27, %r45;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd28, %r46;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p21, %rd31, %rd30;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd31; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd31;
; SM70-NOFTZ-ALLOW-NEXT:    @%p21 bra $L__BB259_5;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd32; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r47, %rd32;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB259_7: // %atomicrmw.start20
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p22, %r47, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r33, %r47, %r3, %p22;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p23, %r47, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r34, %r47, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r35, 0f7FC00000, %r34, %p23;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p24, %r47, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r36, %r33, %r35, %p24;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd33, %r36;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p25, %r48, -2147483648;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r37, %r48, %r4, %p25;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p26, %r48, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r38, %r48, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r39, 0f7FC00000, %r38, %p26;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p27, %r48, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r40, %r37, %r39, %p27;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd34, %r40;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd37, %r47;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd38, %r48;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p28, %rd41, %rd40;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd41; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r47, %rd41;
; SM70-NOFTZ-ALLOW-NEXT:    @%p28 bra $L__BB259_7;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r45, %r46, %r47, %r48};
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v8f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<29>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<49>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v8f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB259_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r41, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r9, %r41, %r5, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r41, %r5;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r10, %r41, %r5;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r41, %r5;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p4, %r42, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r13, %r42, %r6, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p5, %r42, %r6;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r14, %r42, %r6;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p5;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p6, %r42, %r6;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p7 bra $L__BB259_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB259_3: // %atomicrmw.start6
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p8, %r43, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r17, %r43, %r7, %p8;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r43, %r7;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r18, %r43, %r7;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p9;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p10, %r43, %r7;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p10;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r20;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p11, %r44, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r21, %r44, %r8, %p11;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p12, %r44, %r8;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r22, %r44, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r23, 0f7FC00000, %r22, %p12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p13, %r44, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r24, %r21, %r23, %p13;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r24;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p14 bra $L__BB259_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd22; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd22;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB259_5: // %atomicrmw.start15
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p15, %r45, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r25, %r45, %r1, %p15;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p16, %r45, %r1;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r26, %r45, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r27, 0f7FC00000, %r26, %p16;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p17, %r45, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r28, %r25, %r27, %p17;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd23, %r28;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p18, %r46, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r29, %r46, %r2, %p18;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p19, %r46, %r2;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r30, %r46, %r2;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r31, 0f7FC00000, %r30, %p19;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p20, %r46, %r2;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r32, %r29, %r31, %p20;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd24, %r32;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd27, %r45;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd28, %r46;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p21, %rd31, %rd30;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd31; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd31;
; SM70-FTZ-DISALLOW-NEXT:    @%p21 bra $L__BB259_5;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd32; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r47, %rd32;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB259_7: // %atomicrmw.start20
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p22, %r47, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r33, %r47, %r3, %p22;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p23, %r47, %r3;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r34, %r47, %r3;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r35, 0f7FC00000, %r34, %p23;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p24, %r47, %r3;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r36, %r33, %r35, %p24;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd33, %r36;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p25, %r48, -2147483648;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r37, %r48, %r4, %p25;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p26, %r48, %r4;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r38, %r48, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r39, 0f7FC00000, %r38, %p26;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p27, %r48, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r40, %r37, %r39, %p27;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd34, %r40;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd37, %r47;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd38, %r48;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p28, %rd41, %rd40;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd41; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r47, %rd41;
; SM70-FTZ-DISALLOW-NEXT:    @%p28 bra $L__BB259_7;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r45, %r46, %r47, %r48};
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_v8f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<29>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<49>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v8f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f32_global_cta_param_1+16];
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fminimum_acq_rel_v8f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB259_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r41, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r9, %r41, %r5, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r41, %r5;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r10, %r41, %r5;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r41, %r5;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p4, %r42, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r13, %r42, %r6, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p5, %r42, %r6;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r14, %r42, %r6;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p5;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p6, %r42, %r6;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p6;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p7 bra $L__BB259_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-FTZ-ALLOW-NEXT:  $L__BB259_3: // %atomicrmw.start6
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p8, %r43, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r17, %r43, %r7, %p8;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r43, %r7;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r18, %r43, %r7;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p9;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p10, %r43, %r7;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p10;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r20;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p11, %r44, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r21, %r44, %r8, %p11;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p12, %r44, %r8;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r22, %r44, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r23, 0f7FC00000, %r22, %p12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p13, %r44, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r24, %r21, %r23, %p13;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r24;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p14 bra $L__BB259_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd22; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd22;
; SM70-FTZ-ALLOW-NEXT:  $L__BB259_5: // %atomicrmw.start15
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p15, %r45, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r25, %r45, %r1, %p15;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p16, %r45, %r1;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r26, %r45, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r27, 0f7FC00000, %r26, %p16;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p17, %r45, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r28, %r25, %r27, %p17;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd23, %r28;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p18, %r46, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r29, %r46, %r2, %p18;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p19, %r46, %r2;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r30, %r46, %r2;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r31, 0f7FC00000, %r30, %p19;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p20, %r46, %r2;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r32, %r29, %r31, %p20;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd24, %r32;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd27, %r45;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd28, %r46;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p21, %rd31, %rd30;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd31; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd31;
; SM70-FTZ-ALLOW-NEXT:    @%p21 bra $L__BB259_5;
; SM70-FTZ-ALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd32; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r47, %rd32;
; SM70-FTZ-ALLOW-NEXT:  $L__BB259_7: // %atomicrmw.start20
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p22, %r47, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r33, %r47, %r3, %p22;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p23, %r47, %r3;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r34, %r47, %r3;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r35, 0f7FC00000, %r34, %p23;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p24, %r47, %r3;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r36, %r33, %r35, %p24;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd33, %r36;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p25, %r48, -2147483648;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r37, %r48, %r4, %p25;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p26, %r48, %r4;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r38, %r48, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r39, 0f7FC00000, %r38, %p26;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p27, %r48, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r40, %r37, %r39, %p27;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd34, %r40;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd37, %r47;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd38, %r48;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p28, %rd41, %rd40;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd41; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r47, %rd41;
; SM70-FTZ-ALLOW-NEXT:    @%p28 bra $L__BB259_7;
; SM70-FTZ-ALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r45, %r46, %r47, %r48};
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
        ret <8 x float> %retval
}

define <1 x float> @fmaximum_acq_rel_v1f32_global_cta(ptr addrspace(1) %addr, <1 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_v1f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<8>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB260_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r7, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r3, %r7, %r1, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r7, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r4, %r7, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p3, %r7, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p4, %r2, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r7, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p4 bra $L__BB260_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v1f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<5>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<8>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB260_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r7, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r3, %r7, %r1, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r7, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r4, %r7, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p3, %r7, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p4, %r2, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r7, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    @%p4 bra $L__BB260_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v1f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<5>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<8>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:  $L__BB260_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r7, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r3, %r7, %r1, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r7, %r1;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r4, %r7, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r7, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p4, %r2, %r7;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r7, %r2;
; SM70-FTZ-DISALLOW-NEXT:    @%p4 bra $L__BB260_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v1f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<5>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<8>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v1f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r1, [fmaximum_acq_rel_v1f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r7, [%rd1];
; SM70-FTZ-ALLOW-NEXT:  $L__BB260_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r7, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r3, %r7, %r1, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r7, %r1;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r4, %r7, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r7, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r7, %r6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p4, %r2, %r7;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r7, %r2;
; SM70-FTZ-ALLOW-NEXT:    @%p4 bra $L__BB260_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <1 x float> %val syncscope("block") acq_rel
        ret <1 x float> %retval
}

define <2 x float> @fmaximum_acq_rel_v2f32_global_cta(ptr addrspace(1) %addr, <2 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_v2f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<8>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v2f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB261_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r11, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r3, %r11, %r1, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r11, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r4, %r11, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p3, %r11, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p4, %r12, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r7, %r12, %r2, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p5, %r12, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r8, %r12, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r9, 0f7FC00000, %r8, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p6, %r12, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r10, %r7, %r9, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p7 bra $L__BB261_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r11, %r12};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v2f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<8>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v2f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB261_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r11, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r3, %r11, %r1, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r11, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r4, %r11, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p3, %r11, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p4, %r12, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r7, %r12, %r2, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p5, %r12, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r8, %r12, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r9, 0f7FC00000, %r8, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p6, %r12, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r10, %r7, %r9, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p7 bra $L__BB261_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r11, %r12};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v2f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<8>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v2f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB261_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r11, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r3, %r11, %r1, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r11, %r1;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r4, %r11, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r11, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r6;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p4, %r12, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r7, %r12, %r2, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p5, %r12, %r2;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r8, %r12, %r2;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r9, 0f7FC00000, %r8, %p5;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p6, %r12, %r2;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r10, %r7, %r9, %p6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r11;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p7 bra $L__BB261_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r11, %r12};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v2f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<8>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v2f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v2f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB261_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r11, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r3, %r11, %r1, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r11, %r1;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r4, %r11, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r5, 0f7FC00000, %r4, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r11, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r6, %r3, %r5, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r6;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p4, %r12, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r7, %r12, %r2, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p5, %r12, %r2;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r8, %r12, %r2;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r9, 0f7FC00000, %r8, %p5;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p6, %r12, %r2;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r10, %r7, %r9, %p6;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r11;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r12;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p7 bra $L__BB261_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r11, %r12};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <2 x float> %val syncscope("block") acq_rel
        ret <2 x float> %retval
}

define <4 x float> @fmaximum_acq_rel_v4f32_global_cta(ptr addrspace(1) %addr, <4 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_v4f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<15>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v4f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB262_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r21, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r5, %r21, %r1, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r21, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r6, %r21, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r7, 0f7FC00000, %r6, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p3, %r21, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r8, %r5, %r7, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p4, %r22, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r9, %r22, %r2, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p5, %r22, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r10, %r22, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p6, %r22, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p7 bra $L__BB262_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB262_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p8, %r23, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r13, %r23, %r3, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p9, %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r14, %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p10, %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p10;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p11, %r24, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r17, %r24, %r4, %p11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p12, %r24, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r18, %r24, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p13, %r24, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p13;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r23;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p14 bra $L__BB262_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v4f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<15>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v4f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB262_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r21, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r5, %r21, %r1, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r21, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r6, %r21, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r7, 0f7FC00000, %r6, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p3, %r21, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r8, %r5, %r7, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p4, %r22, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r9, %r22, %r2, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p5, %r22, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r10, %r22, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p6, %r22, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p7 bra $L__BB262_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB262_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p8, %r23, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r13, %r23, %r3, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p9, %r23, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r14, %r23, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p10, %r23, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p10;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p11, %r24, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r17, %r24, %r4, %p11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p12, %r24, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r18, %r24, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p13, %r24, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p13;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r23;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p14 bra $L__BB262_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v4f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<15>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v4f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB262_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r21, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r5, %r21, %r1, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r21, %r1;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r6, %r21, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r7, 0f7FC00000, %r6, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r21, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r8, %r5, %r7, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r8;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p4, %r22, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r9, %r22, %r2, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p5, %r22, %r2;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r10, %r22, %r2;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p5;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p6, %r22, %r2;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p7 bra $L__BB262_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd12;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB262_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p8, %r23, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r13, %r23, %r3, %p8;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r23, %r3;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r14, %r23, %r3;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p9;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p10, %r23, %r3;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p10;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r16;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p11, %r24, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r17, %r24, %r4, %p11;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p12, %r24, %r4;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r18, %r24, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p13, %r24, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p13;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r20;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r23;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r24;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r23, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p14 bra $L__BB262_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v4f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<15>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<25>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v4f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v4f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB262_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r21, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r5, %r21, %r1, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r21, %r1;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r6, %r21, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r7, 0f7FC00000, %r6, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r21, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r8, %r5, %r7, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r8;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p4, %r22, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r9, %r22, %r2, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p5, %r22, %r2;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r10, %r22, %r2;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p5;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p6, %r22, %r2;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p6;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r12;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p7 bra $L__BB262_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd12;
; SM70-FTZ-ALLOW-NEXT:  $L__BB262_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p8, %r23, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r13, %r23, %r3, %p8;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r23, %r3;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r14, %r23, %r3;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p9;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p10, %r23, %r3;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p10;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r16;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p11, %r24, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r17, %r24, %r4, %p11;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p12, %r24, %r4;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r18, %r24, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p13, %r24, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p13;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r20;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r23;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r24;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r23, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p14 bra $L__BB262_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r21, %r22, %r23, %r24};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <4 x float> %val syncscope("block") acq_rel
        ret <4 x float> %retval
}

define <8 x float> @fmaximum_acq_rel_v8f32_global_cta(ptr addrspace(1) %addr, <8 x float> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_v8f32_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<29>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<49>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v8f32_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB263_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r41, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r9, %r41, %r5, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r41, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r10, %r41, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p3, %r41, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p4, %r42, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r13, %r42, %r6, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p5, %r42, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r14, %r42, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p6, %r42, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p7 bra $L__BB263_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB263_3: // %atomicrmw.start6
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p8, %r43, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r17, %r43, %r7, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p9, %r43, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r18, %r43, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p10, %r43, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p10;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p11, %r44, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r21, %r44, %r8, %p11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p12, %r44, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r22, %r44, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r23, 0f7FC00000, %r22, %p12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p13, %r44, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r24, %r21, %r23, %p13;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p14 bra $L__BB263_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd22; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd22;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB263_5: // %atomicrmw.start15
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p15, %r45, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r25, %r45, %r1, %p15;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p16, %r45, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r26, %r45, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r27, 0f7FC00000, %r26, %p16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p17, %r45, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r28, %r25, %r27, %p17;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd23, %r28;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p18, %r46, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r29, %r46, %r2, %p18;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p19, %r46, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r30, %r46, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r31, 0f7FC00000, %r30, %p19;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p20, %r46, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r32, %r29, %r31, %p20;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd24, %r32;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd27, %r45;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd28, %r46;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p21, %rd31, %rd30;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd31; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd31;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p21 bra $L__BB263_5;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd32; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r47, %rd32;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB263_7: // %atomicrmw.start20
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p22, %r47, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r33, %r47, %r3, %p22;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p23, %r47, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r34, %r47, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r35, 0f7FC00000, %r34, %p23;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p24, %r47, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r36, %r33, %r35, %p24;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd33, %r36;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b32 %p25, %r48, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r37, %r48, %r4, %p25;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p26, %r48, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r38, %r48, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r39, 0f7FC00000, %r38, %p26;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p27, %r48, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.f32 %r40, %r37, %r39, %p27;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd34, %r40;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd37, %r47;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd38, %r48;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p28, %rd41, %rd40;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd41; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r47, %rd41;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p28 bra $L__BB263_7;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r45, %r46, %r47, %r48};
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v8f32_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<29>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<49>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v8f32_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB263_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r41, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r9, %r41, %r5, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r41, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r10, %r41, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p3, %r41, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p4, %r42, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r13, %r42, %r6, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p5, %r42, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r14, %r42, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p6, %r42, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p7 bra $L__BB263_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB263_3: // %atomicrmw.start6
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p8, %r43, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r17, %r43, %r7, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p9, %r43, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r18, %r43, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p10, %r43, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p10;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p11, %r44, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r21, %r44, %r8, %p11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p12, %r44, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r22, %r44, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r23, 0f7FC00000, %r22, %p12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p13, %r44, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r24, %r21, %r23, %p13;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p14 bra $L__BB263_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd22; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd22;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB263_5: // %atomicrmw.start15
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p15, %r45, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r25, %r45, %r1, %p15;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p16, %r45, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r26, %r45, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r27, 0f7FC00000, %r26, %p16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p17, %r45, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r28, %r25, %r27, %p17;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd23, %r28;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p18, %r46, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r29, %r46, %r2, %p18;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p19, %r46, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r30, %r46, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r31, 0f7FC00000, %r30, %p19;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p20, %r46, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r32, %r29, %r31, %p20;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd24, %r32;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd27, %r45;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd28, %r46;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p21, %rd31, %rd30;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd31; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd31;
; SM70-NOFTZ-ALLOW-NEXT:    @%p21 bra $L__BB263_5;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd32; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r47, %rd32;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB263_7: // %atomicrmw.start20
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p22, %r47, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r33, %r47, %r3, %p22;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p23, %r47, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r34, %r47, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r35, 0f7FC00000, %r34, %p23;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p24, %r47, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r36, %r33, %r35, %p24;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd33, %r36;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b32 %p25, %r48, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r37, %r48, %r4, %p25;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p26, %r48, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r38, %r48, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r39, 0f7FC00000, %r38, %p26;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p27, %r48, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.f32 %r40, %r37, %r39, %p27;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd34, %r40;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd37, %r47;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd38, %r48;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p28, %rd41, %rd40;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd41; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r47, %rd41;
; SM70-NOFTZ-ALLOW-NEXT:    @%p28 bra $L__BB263_7;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r45, %r46, %r47, %r48};
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v8f32_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<29>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<49>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v8f32_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB263_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p1, %r41, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r9, %r41, %r5, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r41, %r5;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r10, %r41, %r5;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r41, %r5;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p4, %r42, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r13, %r42, %r6, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p5, %r42, %r6;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r14, %r42, %r6;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p5;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p6, %r42, %r6;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p7 bra $L__BB263_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB263_3: // %atomicrmw.start6
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p8, %r43, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r17, %r43, %r7, %p8;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r43, %r7;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r18, %r43, %r7;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p9;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p10, %r43, %r7;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p10;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r20;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p11, %r44, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r21, %r44, %r8, %p11;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p12, %r44, %r8;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r22, %r44, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r23, 0f7FC00000, %r22, %p12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p13, %r44, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r24, %r21, %r23, %p13;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r24;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p14 bra $L__BB263_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd22; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd22;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB263_5: // %atomicrmw.start15
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p15, %r45, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r25, %r45, %r1, %p15;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p16, %r45, %r1;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r26, %r45, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r27, 0f7FC00000, %r26, %p16;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p17, %r45, %r1;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r28, %r25, %r27, %p17;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd23, %r28;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p18, %r46, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r29, %r46, %r2, %p18;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p19, %r46, %r2;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r30, %r46, %r2;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r31, 0f7FC00000, %r30, %p19;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p20, %r46, %r2;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r32, %r29, %r31, %p20;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd24, %r32;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd27, %r45;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd28, %r46;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p21, %rd31, %rd30;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd31; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r45, %rd31;
; SM70-FTZ-DISALLOW-NEXT:    @%p21 bra $L__BB263_5;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd32; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r47, %rd32;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB263_7: // %atomicrmw.start20
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p22, %r47, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r33, %r47, %r3, %p22;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p23, %r47, %r3;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r34, %r47, %r3;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r35, 0f7FC00000, %r34, %p23;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p24, %r47, %r3;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r36, %r33, %r35, %p24;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd33, %r36;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b32 %p25, %r48, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r37, %r48, %r4, %p25;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p26, %r48, %r4;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r38, %r48, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r39, 0f7FC00000, %r38, %p26;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p27, %r48, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.f32 %r40, %r37, %r39, %p27;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd34, %r40;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd37, %r47;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd38, %r48;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p28, %rd41, %rd40;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd41; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r47, %rd41;
; SM70-FTZ-DISALLOW-NEXT:    @%p28 bra $L__BB263_7;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r45, %r46, %r47, %r48};
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v8f32_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<29>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<49>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<42>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v8f32_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f32_global_cta_param_1+16];
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [fmaximum_acq_rel_v8f32_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB263_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p1, %r41, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r9, %r41, %r5, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r41, %r5;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r10, %r41, %r5;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r11, 0f7FC00000, %r10, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p3, %r41, %r5;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r12, %r9, %r11, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p4, %r42, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r13, %r42, %r6, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p5, %r42, %r6;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r14, %r42, %r6;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r15, 0f7FC00000, %r14, %p5;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p6, %r42, %r6;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r16, %r13, %r15, %p6;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p7, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p7 bra $L__BB263_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-FTZ-ALLOW-NEXT:  $L__BB263_3: // %atomicrmw.start6
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p8, %r43, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r17, %r43, %r7, %p8;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r43, %r7;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r18, %r43, %r7;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r19, 0f7FC00000, %r18, %p9;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p10, %r43, %r7;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r20, %r17, %r19, %p10;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r20;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p11, %r44, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r21, %r44, %r8, %p11;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p12, %r44, %r8;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r22, %r44, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r23, 0f7FC00000, %r22, %p12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p13, %r44, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r24, %r21, %r23, %p13;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r24;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p14, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p14 bra $L__BB263_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd22, [%rd1+16];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd22; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd22;
; SM70-FTZ-ALLOW-NEXT:  $L__BB263_5: // %atomicrmw.start15
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p15, %r45, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r25, %r45, %r1, %p15;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p16, %r45, %r1;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r26, %r45, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r27, 0f7FC00000, %r26, %p16;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p17, %r45, %r1;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r28, %r25, %r27, %p17;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd23, %r28;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p18, %r46, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r29, %r46, %r2, %p18;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p19, %r46, %r2;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r30, %r46, %r2;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r31, 0f7FC00000, %r30, %p19;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p20, %r46, %r2;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r32, %r29, %r31, %p20;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd24, %r32;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd25, %rd24, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd26, %rd23, %rd25;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd27, %r45;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd28, %r46;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd29, %rd28, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd30, %rd27, %rd29;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd1+16], %rd30, %rd26;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p21, %rd31, %rd30;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r46}, %rd31; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r45, %rd31;
; SM70-FTZ-ALLOW-NEXT:    @%p21 bra $L__BB263_5;
; SM70-FTZ-ALLOW-NEXT:  // %bb.6: // %atomicrmw.end14
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd1+24];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd32; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r47, %rd32;
; SM70-FTZ-ALLOW-NEXT:  $L__BB263_7: // %atomicrmw.start20
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p22, %r47, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r33, %r47, %r3, %p22;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p23, %r47, %r3;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r34, %r47, %r3;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r35, 0f7FC00000, %r34, %p23;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p24, %r47, %r3;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r36, %r33, %r35, %p24;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd33, %r36;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b32 %p25, %r48, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r37, %r48, %r4, %p25;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p26, %r48, %r4;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r38, %r48, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r39, 0f7FC00000, %r38, %p26;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p27, %r48, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.f32 %r40, %r37, %r39, %p27;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd34, %r40;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd35, %rd34, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd36, %rd33, %rd35;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd37, %r47;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd38, %r48;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd39, %rd38, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd40, %rd37, %rd39;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd41, [%rd1+24], %rd40, %rd36;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p28, %rd41, %rd40;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r48}, %rd41; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r47, %rd41;
; SM70-FTZ-ALLOW-NEXT:    @%p28 bra $L__BB263_7;
; SM70-FTZ-ALLOW-NEXT:  // %bb.8: // %atomicrmw.end19
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0+16], {%r45, %r46, %r47, %r48};
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <8 x float> %val syncscope("block") acq_rel
        ret <8 x float> %retval
}

define <1 x double> @fadd_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
; SM70-LABEL: fadd_acq_rel_v1f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<4>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v1f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd2, [fadd_acq_rel_v1f64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd3, [%rd1], %rd2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd3;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
        ret <1 x double> %retval
}

define <2 x double> @fadd_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM70-LABEL: fadd_acq_rel_v2f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v2f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd4, [%rd1], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd5, [%rd1+8], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd4, %rd5};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
        ret <2 x double> %retval
}

define <4 x double> @fadd_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
; SM70-LABEL: fadd_acq_rel_v4f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<10>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v4f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v4f64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd6, [%rd1], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd7, [%rd1+8], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd8, [%rd1+16], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd9, [%rd1+24], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd8, %rd9};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd6, %rd7};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
        ret <4 x double> %retval
}

define <8 x double> @fadd_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
; SM70-LABEL: fadd_acq_rel_v8f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<18>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v8f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [fadd_acq_rel_v8f64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [fadd_acq_rel_v8f64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd6, %rd7}, [fadd_acq_rel_v8f64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [fadd_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd10, [%rd1], %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd11, [%rd1+8], %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd12, [%rd1+16], %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd13, [%rd1+24], %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd14, [%rd1+32], %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd15, [%rd1+40], %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd16, [%rd1+48], %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd17, [%rd1+56], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd16, %rd17};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd14, %rd15};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd12, %rd13};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd10, %rd11};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
        ret <8 x double> %retval
}

define <1 x double> @fsub_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
; SM70-LABEL: fsub_acq_rel_v1f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<5>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v1f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd2, [fsub_acq_rel_v1f64_global_cta_param_1];
; SM70-NEXT:    neg.f64 %rd3, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd4, [%rd1], %rd3;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd4;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
        ret <1 x double> %retval
}

define <2 x double> @fsub_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM70-LABEL: fsub_acq_rel_v2f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<8>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v2f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [fsub_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT:    neg.f64 %rd4, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd5, [%rd1], %rd4;
; SM70-NEXT:    neg.f64 %rd6, %rd3;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd7, [%rd1+8], %rd6;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd5, %rd7};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
        ret <2 x double> %retval
}

define <4 x double> @fsub_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
; SM70-LABEL: fsub_acq_rel_v4f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<14>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v4f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [fsub_acq_rel_v4f64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [fsub_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT:    neg.f64 %rd6, %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd7, [%rd1], %rd6;
; SM70-NEXT:    neg.f64 %rd8, %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd9, [%rd1+8], %rd8;
; SM70-NEXT:    neg.f64 %rd10, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd11, [%rd1+16], %rd10;
; SM70-NEXT:    neg.f64 %rd12, %rd3;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd13, [%rd1+24], %rd12;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd11, %rd13};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd7, %rd9};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
        ret <4 x double> %retval
}

define <8 x double> @fsub_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
; SM70-LABEL: fsub_acq_rel_v8f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b64 %rd<26>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v8f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [fsub_acq_rel_v8f64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd5}, [fsub_acq_rel_v8f64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd6, %rd7}, [fsub_acq_rel_v8f64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [fsub_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT:    neg.f64 %rd10, %rd8;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd11, [%rd1], %rd10;
; SM70-NEXT:    neg.f64 %rd12, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd13, [%rd1+8], %rd12;
; SM70-NEXT:    neg.f64 %rd14, %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd15, [%rd1+16], %rd14;
; SM70-NEXT:    neg.f64 %rd16, %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd17, [%rd1+24], %rd16;
; SM70-NEXT:    neg.f64 %rd18, %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd19, [%rd1+32], %rd18;
; SM70-NEXT:    neg.f64 %rd20, %rd5;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd21, [%rd1+40], %rd20;
; SM70-NEXT:    neg.f64 %rd22, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd23, [%rd1+48], %rd22;
; SM70-NEXT:    neg.f64 %rd24, %rd3;
; SM70-NEXT:    atom.relaxed.cta.global.add.f64 %rd25, [%rd1+56], %rd24;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd23, %rd25};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd19, %rd21};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd15, %rd17};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd11, %rd13};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
        ret <8 x double> %retval
}

define <1 x double> @fmin_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
; SM70-LABEL: fmin_acq_rel_v1f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd3, [fmin_acq_rel_v1f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v1f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd5, [%rd3];
; SM70-NEXT:  $L__BB272_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    min.f64 %rd4, %rd5, %rd1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
; SM70-NEXT:    setp.ne.b64 %p1, %rd2, %rd5;
; SM70-NEXT:    mov.b64 %rd5, %rd2;
; SM70-NEXT:    @%p1 bra $L__BB272_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
        ret <1 x double> %retval
}

define <2 x double> @fmin_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM70-LABEL: fmin_acq_rel_v2f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b64 %rd<10>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd5, [fmin_acq_rel_v2f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd1}, [fmin_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd8, [%rd5];
; SM70-NEXT:  $L__BB273_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd3, %rd8;
; SM70-NEXT:    min.f64 %rd6, %rd3, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd8, [%rd5], %rd3, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd8, %rd3;
; SM70-NEXT:    @%p1 bra $L__BB273_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd9, [%rd5+8];
; SM70-NEXT:  $L__BB273_3: // %atomicrmw.start4
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    min.f64 %rd7, %rd9, %rd1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
; SM70-NEXT:    setp.ne.b64 %p2, %rd4, %rd9;
; SM70-NEXT:    mov.b64 %rd9, %rd4;
; SM70-NEXT:    @%p2 bra $L__BB273_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end3
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd8, %rd4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
        ret <2 x double> %retval
}

define <4 x double> @fmin_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
; SM70-LABEL: fmin_acq_rel_v4f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<5>;
; SM70-NEXT:    .reg .b64 %rd<18>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd9, [fmin_acq_rel_v4f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd6}, [fmin_acq_rel_v4f64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd3, %rd2}, [fmin_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd14, [%rd9];
; SM70-NEXT:  $L__BB274_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd4, %rd14;
; SM70-NEXT:    min.f64 %rd10, %rd4, %rd3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd4, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd14, %rd4;
; SM70-NEXT:    @%p1 bra $L__BB274_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd15, [%rd9+8];
; SM70-NEXT:  $L__BB274_3: // %atomicrmw.start8
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd15;
; SM70-NEXT:    min.f64 %rd11, %rd5, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd5, %rd11;
; SM70-NEXT:    setp.ne.b64 %p2, %rd15, %rd5;
; SM70-NEXT:    @%p2 bra $L__BB274_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end7
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd16, [%rd9+16];
; SM70-NEXT:  $L__BB274_5: // %atomicrmw.start19
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd7, %rd16;
; SM70-NEXT:    min.f64 %rd12, %rd7, %rd1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd16, [%rd9+16], %rd7, %rd12;
; SM70-NEXT:    setp.ne.b64 %p3, %rd16, %rd7;
; SM70-NEXT:    @%p3 bra $L__BB274_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end18
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd17, [%rd9+24];
; SM70-NEXT:  $L__BB274_7: // %atomicrmw.start26
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    min.f64 %rd13, %rd17, %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
; SM70-NEXT:    setp.ne.b64 %p4, %rd8, %rd17;
; SM70-NEXT:    mov.b64 %rd17, %rd8;
; SM70-NEXT:    @%p4 bra $L__BB274_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end25
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd16, %rd8};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd14, %rd15};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
        ret <4 x double> %retval
}

define <8 x double> @fmin_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
; SM70-LABEL: fmin_acq_rel_v8f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<9>;
; SM70-NEXT:    .reg .b64 %rd<34>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd17, [fmin_acq_rel_v8f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd10, %rd14}, [fmin_acq_rel_v8f64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd11}, [fmin_acq_rel_v8f64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd7}, [fmin_acq_rel_v8f64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd3}, [fmin_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd26, [%rd17];
; SM70-NEXT:  $L__BB275_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd26;
; SM70-NEXT:    min.f64 %rd18, %rd5, %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd5, %rd18;
; SM70-NEXT:    setp.ne.b64 %p1, %rd26, %rd5;
; SM70-NEXT:    @%p1 bra $L__BB275_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd27, [%rd17+8];
; SM70-NEXT:  $L__BB275_3: // %atomicrmw.start12
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd6, %rd27;
; SM70-NEXT:    min.f64 %rd19, %rd6, %rd3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd6, %rd19;
; SM70-NEXT:    setp.ne.b64 %p2, %rd27, %rd6;
; SM70-NEXT:    @%p2 bra $L__BB275_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end11
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd28, [%rd17+16];
; SM70-NEXT:  $L__BB275_5: // %atomicrmw.start23
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd8, %rd28;
; SM70-NEXT:    min.f64 %rd20, %rd8, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd8, %rd20;
; SM70-NEXT:    setp.ne.b64 %p3, %rd28, %rd8;
; SM70-NEXT:    @%p3 bra $L__BB275_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end22
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd29, [%rd17+24];
; SM70-NEXT:  $L__BB275_7: // %atomicrmw.start30
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd9, %rd29;
; SM70-NEXT:    min.f64 %rd21, %rd9, %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd9, %rd21;
; SM70-NEXT:    setp.ne.b64 %p4, %rd29, %rd9;
; SM70-NEXT:    @%p4 bra $L__BB275_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end29
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd30, [%rd17+32];
; SM70-NEXT:  $L__BB275_9: // %atomicrmw.start45
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd12, %rd30;
; SM70-NEXT:    min.f64 %rd22, %rd12, %rd1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd12, %rd22;
; SM70-NEXT:    setp.ne.b64 %p5, %rd30, %rd12;
; SM70-NEXT:    @%p5 bra $L__BB275_9;
; SM70-NEXT:  // %bb.10: // %atomicrmw.end44
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd31, [%rd17+40];
; SM70-NEXT:  $L__BB275_11: // %atomicrmw.start52
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd13, %rd31;
; SM70-NEXT:    min.f64 %rd23, %rd13, %rd11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd13, %rd23;
; SM70-NEXT:    setp.ne.b64 %p6, %rd31, %rd13;
; SM70-NEXT:    @%p6 bra $L__BB275_11;
; SM70-NEXT:  // %bb.12: // %atomicrmw.end51
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd17+48];
; SM70-NEXT:  $L__BB275_13: // %atomicrmw.start63
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd15, %rd32;
; SM70-NEXT:    min.f64 %rd24, %rd15, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd32, [%rd17+48], %rd15, %rd24;
; SM70-NEXT:    setp.ne.b64 %p7, %rd32, %rd15;
; SM70-NEXT:    @%p7 bra $L__BB275_13;
; SM70-NEXT:  // %bb.14: // %atomicrmw.end62
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd33, [%rd17+56];
; SM70-NEXT:  $L__BB275_15: // %atomicrmw.start70
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    min.f64 %rd25, %rd33, %rd14;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
; SM70-NEXT:    setp.ne.b64 %p8, %rd16, %rd33;
; SM70-NEXT:    mov.b64 %rd33, %rd16;
; SM70-NEXT:    @%p8 bra $L__BB275_15;
; SM70-NEXT:  // %bb.16: // %atomicrmw.end69
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd32, %rd16};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
        ret <8 x double> %retval
}

define <1 x double> @fmax_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
; SM70-LABEL: fmax_acq_rel_v1f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b64 %rd<6>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd3, [fmax_acq_rel_v1f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v1f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd5, [%rd3];
; SM70-NEXT:  $L__BB276_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.f64 %rd4, %rd5, %rd1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd5, %rd4;
; SM70-NEXT:    setp.ne.b64 %p1, %rd2, %rd5;
; SM70-NEXT:    mov.b64 %rd5, %rd2;
; SM70-NEXT:    @%p1 bra $L__BB276_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
        ret <1 x double> %retval
}

define <2 x double> @fmax_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM70-LABEL: fmax_acq_rel_v2f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b64 %rd<10>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd5, [fmax_acq_rel_v2f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd1}, [fmax_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd8, [%rd5];
; SM70-NEXT:  $L__BB277_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd3, %rd8;
; SM70-NEXT:    max.f64 %rd6, %rd3, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd8, [%rd5], %rd3, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd8, %rd3;
; SM70-NEXT:    @%p1 bra $L__BB277_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd9, [%rd5+8];
; SM70-NEXT:  $L__BB277_3: // %atomicrmw.start4
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.f64 %rd7, %rd9, %rd1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd9, %rd7;
; SM70-NEXT:    setp.ne.b64 %p2, %rd4, %rd9;
; SM70-NEXT:    mov.b64 %rd9, %rd4;
; SM70-NEXT:    @%p2 bra $L__BB277_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end3
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd8, %rd4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
        ret <2 x double> %retval
}

define <4 x double> @fmax_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
; SM70-LABEL: fmax_acq_rel_v4f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<5>;
; SM70-NEXT:    .reg .b64 %rd<18>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd9, [fmax_acq_rel_v4f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd6}, [fmax_acq_rel_v4f64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd3, %rd2}, [fmax_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd14, [%rd9];
; SM70-NEXT:  $L__BB278_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd4, %rd14;
; SM70-NEXT:    max.f64 %rd10, %rd4, %rd3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd14, [%rd9], %rd4, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd14, %rd4;
; SM70-NEXT:    @%p1 bra $L__BB278_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd15, [%rd9+8];
; SM70-NEXT:  $L__BB278_3: // %atomicrmw.start8
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd15;
; SM70-NEXT:    max.f64 %rd11, %rd5, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd15, [%rd9+8], %rd5, %rd11;
; SM70-NEXT:    setp.ne.b64 %p2, %rd15, %rd5;
; SM70-NEXT:    @%p2 bra $L__BB278_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end7
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd16, [%rd9+16];
; SM70-NEXT:  $L__BB278_5: // %atomicrmw.start19
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd7, %rd16;
; SM70-NEXT:    max.f64 %rd12, %rd7, %rd1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd16, [%rd9+16], %rd7, %rd12;
; SM70-NEXT:    setp.ne.b64 %p3, %rd16, %rd7;
; SM70-NEXT:    @%p3 bra $L__BB278_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end18
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd17, [%rd9+24];
; SM70-NEXT:  $L__BB278_7: // %atomicrmw.start26
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.f64 %rd13, %rd17, %rd6;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd17, %rd13;
; SM70-NEXT:    setp.ne.b64 %p4, %rd8, %rd17;
; SM70-NEXT:    mov.b64 %rd17, %rd8;
; SM70-NEXT:    @%p4 bra $L__BB278_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end25
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd16, %rd8};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd14, %rd15};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
        ret <4 x double> %retval
}

define <8 x double> @fmax_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
; SM70-LABEL: fmax_acq_rel_v8f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<9>;
; SM70-NEXT:    .reg .b64 %rd<34>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd17, [fmax_acq_rel_v8f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd10, %rd14}, [fmax_acq_rel_v8f64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd11}, [fmax_acq_rel_v8f64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd7}, [fmax_acq_rel_v8f64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd3}, [fmax_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd26, [%rd17];
; SM70-NEXT:  $L__BB279_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd26;
; SM70-NEXT:    max.f64 %rd18, %rd5, %rd4;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd26, [%rd17], %rd5, %rd18;
; SM70-NEXT:    setp.ne.b64 %p1, %rd26, %rd5;
; SM70-NEXT:    @%p1 bra $L__BB279_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd27, [%rd17+8];
; SM70-NEXT:  $L__BB279_3: // %atomicrmw.start12
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd6, %rd27;
; SM70-NEXT:    max.f64 %rd19, %rd6, %rd3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd27, [%rd17+8], %rd6, %rd19;
; SM70-NEXT:    setp.ne.b64 %p2, %rd27, %rd6;
; SM70-NEXT:    @%p2 bra $L__BB279_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end11
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd28, [%rd17+16];
; SM70-NEXT:  $L__BB279_5: // %atomicrmw.start23
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd8, %rd28;
; SM70-NEXT:    max.f64 %rd20, %rd8, %rd2;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd28, [%rd17+16], %rd8, %rd20;
; SM70-NEXT:    setp.ne.b64 %p3, %rd28, %rd8;
; SM70-NEXT:    @%p3 bra $L__BB279_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end22
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd29, [%rd17+24];
; SM70-NEXT:  $L__BB279_7: // %atomicrmw.start30
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd9, %rd29;
; SM70-NEXT:    max.f64 %rd21, %rd9, %rd7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd29, [%rd17+24], %rd9, %rd21;
; SM70-NEXT:    setp.ne.b64 %p4, %rd29, %rd9;
; SM70-NEXT:    @%p4 bra $L__BB279_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end29
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd30, [%rd17+32];
; SM70-NEXT:  $L__BB279_9: // %atomicrmw.start45
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd12, %rd30;
; SM70-NEXT:    max.f64 %rd22, %rd12, %rd1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd30, [%rd17+32], %rd12, %rd22;
; SM70-NEXT:    setp.ne.b64 %p5, %rd30, %rd12;
; SM70-NEXT:    @%p5 bra $L__BB279_9;
; SM70-NEXT:  // %bb.10: // %atomicrmw.end44
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd31, [%rd17+40];
; SM70-NEXT:  $L__BB279_11: // %atomicrmw.start52
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd13, %rd31;
; SM70-NEXT:    max.f64 %rd23, %rd13, %rd11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd31, [%rd17+40], %rd13, %rd23;
; SM70-NEXT:    setp.ne.b64 %p6, %rd31, %rd13;
; SM70-NEXT:    @%p6 bra $L__BB279_11;
; SM70-NEXT:  // %bb.12: // %atomicrmw.end51
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd32, [%rd17+48];
; SM70-NEXT:  $L__BB279_13: // %atomicrmw.start63
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd15, %rd32;
; SM70-NEXT:    max.f64 %rd24, %rd15, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd32, [%rd17+48], %rd15, %rd24;
; SM70-NEXT:    setp.ne.b64 %p7, %rd32, %rd15;
; SM70-NEXT:    @%p7 bra $L__BB279_13;
; SM70-NEXT:  // %bb.14: // %atomicrmw.end62
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd33, [%rd17+56];
; SM70-NEXT:  $L__BB279_15: // %atomicrmw.start70
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    max.f64 %rd25, %rd33, %rd14;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd33, %rd25;
; SM70-NEXT:    setp.ne.b64 %p8, %rd16, %rd33;
; SM70-NEXT:    mov.b64 %rd33, %rd16;
; SM70-NEXT:    @%p8 bra $L__BB279_15;
; SM70-NEXT:  // %bb.16: // %atomicrmw.end69
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd32, %rd16};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd30, %rd31};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd28, %rd29};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
        ret <8 x double> %retval
}

define <1 x double> @fminimum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
; SM70-LABEL: fminimum_acq_rel_v1f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<5>;
; SM70-NEXT:    .reg .b64 %rd<9>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd3, [fminimum_acq_rel_v1f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v1f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd8, [%rd3];
; SM70-NEXT:  $L__BB280_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    setp.eq.b64 %p1, %rd8, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd4, %rd8, %rd1, %p1;
; SM70-NEXT:    setp.nan.f64 %p2, %rd8, %rd1;
; SM70-NEXT:    min.f64 %rd5, %rd8, %rd1;
; SM70-NEXT:    selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
; SM70-NEXT:    setp.eq.f64 %p3, %rd8, %rd1;
; SM70-NEXT:    selp.f64 %rd7, %rd4, %rd6, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
; SM70-NEXT:    setp.ne.b64 %p4, %rd2, %rd8;
; SM70-NEXT:    mov.b64 %rd8, %rd2;
; SM70-NEXT:    @%p4 bra $L__BB280_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
        ret <1 x double> %retval
}

define <2 x double> @fminimum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM70-LABEL: fminimum_acq_rel_v2f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<9>;
; SM70-NEXT:    .reg .b64 %rd<16>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd5, [fminimum_acq_rel_v2f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd1}, [fminimum_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd14, [%rd5];
; SM70-NEXT:  $L__BB281_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd3, %rd14;
; SM70-NEXT:    setp.eq.b64 %p1, %rd3, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd6, %rd3, %rd2, %p1;
; SM70-NEXT:    setp.nan.f64 %p2, %rd3, %rd2;
; SM70-NEXT:    min.f64 %rd7, %rd3, %rd2;
; SM70-NEXT:    selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p2;
; SM70-NEXT:    setp.eq.f64 %p3, %rd3, %rd2;
; SM70-NEXT:    selp.f64 %rd9, %rd6, %rd8, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd14, [%rd5], %rd3, %rd9;
; SM70-NEXT:    setp.ne.b64 %p4, %rd14, %rd3;
; SM70-NEXT:    @%p4 bra $L__BB281_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd15, [%rd5+8];
; SM70-NEXT:  $L__BB281_3: // %atomicrmw.start4
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    setp.eq.b64 %p5, %rd15, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd10, %rd15, %rd1, %p5;
; SM70-NEXT:    setp.nan.f64 %p6, %rd15, %rd1;
; SM70-NEXT:    min.f64 %rd11, %rd15, %rd1;
; SM70-NEXT:    selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
; SM70-NEXT:    setp.eq.f64 %p7, %rd15, %rd1;
; SM70-NEXT:    selp.f64 %rd13, %rd10, %rd12, %p7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd15, %rd13;
; SM70-NEXT:    setp.ne.b64 %p8, %rd4, %rd15;
; SM70-NEXT:    mov.b64 %rd15, %rd4;
; SM70-NEXT:    @%p8 bra $L__BB281_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end3
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd14, %rd4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
        ret <2 x double> %retval
}

define <4 x double> @fminimum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
; SM70-LABEL: fminimum_acq_rel_v4f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<17>;
; SM70-NEXT:    .reg .b64 %rd<30>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd9, [fminimum_acq_rel_v4f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd6}, [fminimum_acq_rel_v4f64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd3, %rd2}, [fminimum_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd26, [%rd9];
; SM70-NEXT:  $L__BB282_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd4, %rd26;
; SM70-NEXT:    setp.eq.b64 %p1, %rd4, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd10, %rd4, %rd3, %p1;
; SM70-NEXT:    setp.nan.f64 %p2, %rd4, %rd3;
; SM70-NEXT:    min.f64 %rd11, %rd4, %rd3;
; SM70-NEXT:    selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p2;
; SM70-NEXT:    setp.eq.f64 %p3, %rd4, %rd3;
; SM70-NEXT:    selp.f64 %rd13, %rd10, %rd12, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd26, [%rd9], %rd4, %rd13;
; SM70-NEXT:    setp.ne.b64 %p4, %rd26, %rd4;
; SM70-NEXT:    @%p4 bra $L__BB282_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd27, [%rd9+8];
; SM70-NEXT:  $L__BB282_3: // %atomicrmw.start8
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd27;
; SM70-NEXT:    setp.eq.b64 %p5, %rd5, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd14, %rd5, %rd2, %p5;
; SM70-NEXT:    setp.nan.f64 %p6, %rd5, %rd2;
; SM70-NEXT:    min.f64 %rd15, %rd5, %rd2;
; SM70-NEXT:    selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
; SM70-NEXT:    setp.eq.f64 %p7, %rd5, %rd2;
; SM70-NEXT:    selp.f64 %rd17, %rd14, %rd16, %p7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd27, [%rd9+8], %rd5, %rd17;
; SM70-NEXT:    setp.ne.b64 %p8, %rd27, %rd5;
; SM70-NEXT:    @%p8 bra $L__BB282_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end7
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd28, [%rd9+16];
; SM70-NEXT:  $L__BB282_5: // %atomicrmw.start19
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd7, %rd28;
; SM70-NEXT:    setp.eq.b64 %p9, %rd7, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd18, %rd7, %rd1, %p9;
; SM70-NEXT:    setp.nan.f64 %p10, %rd7, %rd1;
; SM70-NEXT:    min.f64 %rd19, %rd7, %rd1;
; SM70-NEXT:    selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p10;
; SM70-NEXT:    setp.eq.f64 %p11, %rd7, %rd1;
; SM70-NEXT:    selp.f64 %rd21, %rd18, %rd20, %p11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd28, [%rd9+16], %rd7, %rd21;
; SM70-NEXT:    setp.ne.b64 %p12, %rd28, %rd7;
; SM70-NEXT:    @%p12 bra $L__BB282_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end18
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd29, [%rd9+24];
; SM70-NEXT:  $L__BB282_7: // %atomicrmw.start26
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    setp.eq.b64 %p13, %rd29, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd22, %rd29, %rd6, %p13;
; SM70-NEXT:    setp.nan.f64 %p14, %rd29, %rd6;
; SM70-NEXT:    min.f64 %rd23, %rd29, %rd6;
; SM70-NEXT:    selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p14;
; SM70-NEXT:    setp.eq.f64 %p15, %rd29, %rd6;
; SM70-NEXT:    selp.f64 %rd25, %rd22, %rd24, %p15;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd29, %rd25;
; SM70-NEXT:    setp.ne.b64 %p16, %rd8, %rd29;
; SM70-NEXT:    mov.b64 %rd29, %rd8;
; SM70-NEXT:    @%p16 bra $L__BB282_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end25
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd28, %rd8};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
        ret <4 x double> %retval
}

define <8 x double> @fminimum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
; SM70-LABEL: fminimum_acq_rel_v8f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<33>;
; SM70-NEXT:    .reg .b64 %rd<58>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd17, [fminimum_acq_rel_v8f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd10, %rd14}, [fminimum_acq_rel_v8f64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd11}, [fminimum_acq_rel_v8f64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd7}, [fminimum_acq_rel_v8f64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd3}, [fminimum_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd50, [%rd17];
; SM70-NEXT:  $L__BB283_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd50;
; SM70-NEXT:    setp.eq.b64 %p1, %rd5, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd18, %rd5, %rd4, %p1;
; SM70-NEXT:    setp.nan.f64 %p2, %rd5, %rd4;
; SM70-NEXT:    min.f64 %rd19, %rd5, %rd4;
; SM70-NEXT:    selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p2;
; SM70-NEXT:    setp.eq.f64 %p3, %rd5, %rd4;
; SM70-NEXT:    selp.f64 %rd21, %rd18, %rd20, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd50, [%rd17], %rd5, %rd21;
; SM70-NEXT:    setp.ne.b64 %p4, %rd50, %rd5;
; SM70-NEXT:    @%p4 bra $L__BB283_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd51, [%rd17+8];
; SM70-NEXT:  $L__BB283_3: // %atomicrmw.start12
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd6, %rd51;
; SM70-NEXT:    setp.eq.b64 %p5, %rd6, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd22, %rd6, %rd3, %p5;
; SM70-NEXT:    setp.nan.f64 %p6, %rd6, %rd3;
; SM70-NEXT:    min.f64 %rd23, %rd6, %rd3;
; SM70-NEXT:    selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
; SM70-NEXT:    setp.eq.f64 %p7, %rd6, %rd3;
; SM70-NEXT:    selp.f64 %rd25, %rd22, %rd24, %p7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd51, [%rd17+8], %rd6, %rd25;
; SM70-NEXT:    setp.ne.b64 %p8, %rd51, %rd6;
; SM70-NEXT:    @%p8 bra $L__BB283_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end11
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd52, [%rd17+16];
; SM70-NEXT:  $L__BB283_5: // %atomicrmw.start23
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd8, %rd52;
; SM70-NEXT:    setp.eq.b64 %p9, %rd8, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd26, %rd8, %rd2, %p9;
; SM70-NEXT:    setp.nan.f64 %p10, %rd8, %rd2;
; SM70-NEXT:    min.f64 %rd27, %rd8, %rd2;
; SM70-NEXT:    selp.f64 %rd28, 0d7FF8000000000000, %rd27, %p10;
; SM70-NEXT:    setp.eq.f64 %p11, %rd8, %rd2;
; SM70-NEXT:    selp.f64 %rd29, %rd26, %rd28, %p11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd52, [%rd17+16], %rd8, %rd29;
; SM70-NEXT:    setp.ne.b64 %p12, %rd52, %rd8;
; SM70-NEXT:    @%p12 bra $L__BB283_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end22
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd53, [%rd17+24];
; SM70-NEXT:  $L__BB283_7: // %atomicrmw.start30
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd9, %rd53;
; SM70-NEXT:    setp.eq.b64 %p13, %rd9, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd30, %rd9, %rd7, %p13;
; SM70-NEXT:    setp.nan.f64 %p14, %rd9, %rd7;
; SM70-NEXT:    min.f64 %rd31, %rd9, %rd7;
; SM70-NEXT:    selp.f64 %rd32, 0d7FF8000000000000, %rd31, %p14;
; SM70-NEXT:    setp.eq.f64 %p15, %rd9, %rd7;
; SM70-NEXT:    selp.f64 %rd33, %rd30, %rd32, %p15;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd53, [%rd17+24], %rd9, %rd33;
; SM70-NEXT:    setp.ne.b64 %p16, %rd53, %rd9;
; SM70-NEXT:    @%p16 bra $L__BB283_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end29
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd54, [%rd17+32];
; SM70-NEXT:  $L__BB283_9: // %atomicrmw.start45
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd12, %rd54;
; SM70-NEXT:    setp.eq.b64 %p17, %rd12, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd34, %rd12, %rd1, %p17;
; SM70-NEXT:    setp.nan.f64 %p18, %rd12, %rd1;
; SM70-NEXT:    min.f64 %rd35, %rd12, %rd1;
; SM70-NEXT:    selp.f64 %rd36, 0d7FF8000000000000, %rd35, %p18;
; SM70-NEXT:    setp.eq.f64 %p19, %rd12, %rd1;
; SM70-NEXT:    selp.f64 %rd37, %rd34, %rd36, %p19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd54, [%rd17+32], %rd12, %rd37;
; SM70-NEXT:    setp.ne.b64 %p20, %rd54, %rd12;
; SM70-NEXT:    @%p20 bra $L__BB283_9;
; SM70-NEXT:  // %bb.10: // %atomicrmw.end44
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd55, [%rd17+40];
; SM70-NEXT:  $L__BB283_11: // %atomicrmw.start52
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd13, %rd55;
; SM70-NEXT:    setp.eq.b64 %p21, %rd13, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd38, %rd13, %rd11, %p21;
; SM70-NEXT:    setp.nan.f64 %p22, %rd13, %rd11;
; SM70-NEXT:    min.f64 %rd39, %rd13, %rd11;
; SM70-NEXT:    selp.f64 %rd40, 0d7FF8000000000000, %rd39, %p22;
; SM70-NEXT:    setp.eq.f64 %p23, %rd13, %rd11;
; SM70-NEXT:    selp.f64 %rd41, %rd38, %rd40, %p23;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd55, [%rd17+40], %rd13, %rd41;
; SM70-NEXT:    setp.ne.b64 %p24, %rd55, %rd13;
; SM70-NEXT:    @%p24 bra $L__BB283_11;
; SM70-NEXT:  // %bb.12: // %atomicrmw.end51
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd56, [%rd17+48];
; SM70-NEXT:  $L__BB283_13: // %atomicrmw.start63
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd15, %rd56;
; SM70-NEXT:    setp.eq.b64 %p25, %rd15, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd42, %rd15, %rd10, %p25;
; SM70-NEXT:    setp.nan.f64 %p26, %rd15, %rd10;
; SM70-NEXT:    min.f64 %rd43, %rd15, %rd10;
; SM70-NEXT:    selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
; SM70-NEXT:    setp.eq.f64 %p27, %rd15, %rd10;
; SM70-NEXT:    selp.f64 %rd45, %rd42, %rd44, %p27;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd56, [%rd17+48], %rd15, %rd45;
; SM70-NEXT:    setp.ne.b64 %p28, %rd56, %rd15;
; SM70-NEXT:    @%p28 bra $L__BB283_13;
; SM70-NEXT:  // %bb.14: // %atomicrmw.end62
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd57, [%rd17+56];
; SM70-NEXT:  $L__BB283_15: // %atomicrmw.start70
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    setp.eq.b64 %p29, %rd57, -9223372036854775808;
; SM70-NEXT:    selp.f64 %rd46, %rd57, %rd14, %p29;
; SM70-NEXT:    setp.nan.f64 %p30, %rd57, %rd14;
; SM70-NEXT:    min.f64 %rd47, %rd57, %rd14;
; SM70-NEXT:    selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p30;
; SM70-NEXT:    setp.eq.f64 %p31, %rd57, %rd14;
; SM70-NEXT:    selp.f64 %rd49, %rd46, %rd48, %p31;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd57, %rd49;
; SM70-NEXT:    setp.ne.b64 %p32, %rd16, %rd57;
; SM70-NEXT:    mov.b64 %rd57, %rd16;
; SM70-NEXT:    @%p32 bra $L__BB283_15;
; SM70-NEXT:  // %bb.16: // %atomicrmw.end69
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd56, %rd16};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd54, %rd55};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd50, %rd51};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
        ret <8 x double> %retval
}

define <1 x double> @fmaximum_acq_rel_v1f64_global_cta(ptr addrspace(1) %addr, <1 x double> %val) {
; SM70-LABEL: fmaximum_acq_rel_v1f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<5>;
; SM70-NEXT:    .reg .b64 %rd<9>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd3, [fmaximum_acq_rel_v1f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v1f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd8, [%rd3];
; SM70-NEXT:  $L__BB284_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    setp.eq.b64 %p1, %rd8, 0;
; SM70-NEXT:    selp.f64 %rd4, %rd8, %rd1, %p1;
; SM70-NEXT:    setp.nan.f64 %p2, %rd8, %rd1;
; SM70-NEXT:    max.f64 %rd5, %rd8, %rd1;
; SM70-NEXT:    selp.f64 %rd6, 0d7FF8000000000000, %rd5, %p2;
; SM70-NEXT:    setp.eq.f64 %p3, %rd8, %rd1;
; SM70-NEXT:    selp.f64 %rd7, %rd4, %rd6, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd2, [%rd3], %rd8, %rd7;
; SM70-NEXT:    setp.ne.b64 %p4, %rd2, %rd8;
; SM70-NEXT:    mov.b64 %rd8, %rd2;
; SM70-NEXT:    @%p4 bra $L__BB284_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b64 [func_retval0], %rd2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <1 x double> %val syncscope("block") acq_rel
        ret <1 x double> %retval
}

define <2 x double> @fmaximum_acq_rel_v2f64_global_cta(ptr addrspace(1) %addr, <2 x double> %val) {
; SM70-LABEL: fmaximum_acq_rel_v2f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<9>;
; SM70-NEXT:    .reg .b64 %rd<16>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd5, [fmaximum_acq_rel_v2f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd1}, [fmaximum_acq_rel_v2f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd14, [%rd5];
; SM70-NEXT:  $L__BB285_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd3, %rd14;
; SM70-NEXT:    setp.eq.b64 %p1, %rd3, 0;
; SM70-NEXT:    selp.f64 %rd6, %rd3, %rd2, %p1;
; SM70-NEXT:    setp.nan.f64 %p2, %rd3, %rd2;
; SM70-NEXT:    max.f64 %rd7, %rd3, %rd2;
; SM70-NEXT:    selp.f64 %rd8, 0d7FF8000000000000, %rd7, %p2;
; SM70-NEXT:    setp.eq.f64 %p3, %rd3, %rd2;
; SM70-NEXT:    selp.f64 %rd9, %rd6, %rd8, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd14, [%rd5], %rd3, %rd9;
; SM70-NEXT:    setp.ne.b64 %p4, %rd14, %rd3;
; SM70-NEXT:    @%p4 bra $L__BB285_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd15, [%rd5+8];
; SM70-NEXT:  $L__BB285_3: // %atomicrmw.start4
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    setp.eq.b64 %p5, %rd15, 0;
; SM70-NEXT:    selp.f64 %rd10, %rd15, %rd1, %p5;
; SM70-NEXT:    setp.nan.f64 %p6, %rd15, %rd1;
; SM70-NEXT:    max.f64 %rd11, %rd15, %rd1;
; SM70-NEXT:    selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p6;
; SM70-NEXT:    setp.eq.f64 %p7, %rd15, %rd1;
; SM70-NEXT:    selp.f64 %rd13, %rd10, %rd12, %p7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd4, [%rd5+8], %rd15, %rd13;
; SM70-NEXT:    setp.ne.b64 %p8, %rd4, %rd15;
; SM70-NEXT:    mov.b64 %rd15, %rd4;
; SM70-NEXT:    @%p8 bra $L__BB285_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end3
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd14, %rd4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <2 x double> %val syncscope("block") acq_rel
        ret <2 x double> %retval
}

define <4 x double> @fmaximum_acq_rel_v4f64_global_cta(ptr addrspace(1) %addr, <4 x double> %val) {
; SM70-LABEL: fmaximum_acq_rel_v4f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<17>;
; SM70-NEXT:    .reg .b64 %rd<30>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd9, [fmaximum_acq_rel_v4f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd6}, [fmaximum_acq_rel_v4f64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd3, %rd2}, [fmaximum_acq_rel_v4f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd26, [%rd9];
; SM70-NEXT:  $L__BB286_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd4, %rd26;
; SM70-NEXT:    setp.eq.b64 %p1, %rd4, 0;
; SM70-NEXT:    selp.f64 %rd10, %rd4, %rd3, %p1;
; SM70-NEXT:    setp.nan.f64 %p2, %rd4, %rd3;
; SM70-NEXT:    max.f64 %rd11, %rd4, %rd3;
; SM70-NEXT:    selp.f64 %rd12, 0d7FF8000000000000, %rd11, %p2;
; SM70-NEXT:    setp.eq.f64 %p3, %rd4, %rd3;
; SM70-NEXT:    selp.f64 %rd13, %rd10, %rd12, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd26, [%rd9], %rd4, %rd13;
; SM70-NEXT:    setp.ne.b64 %p4, %rd26, %rd4;
; SM70-NEXT:    @%p4 bra $L__BB286_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd27, [%rd9+8];
; SM70-NEXT:  $L__BB286_3: // %atomicrmw.start8
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd27;
; SM70-NEXT:    setp.eq.b64 %p5, %rd5, 0;
; SM70-NEXT:    selp.f64 %rd14, %rd5, %rd2, %p5;
; SM70-NEXT:    setp.nan.f64 %p6, %rd5, %rd2;
; SM70-NEXT:    max.f64 %rd15, %rd5, %rd2;
; SM70-NEXT:    selp.f64 %rd16, 0d7FF8000000000000, %rd15, %p6;
; SM70-NEXT:    setp.eq.f64 %p7, %rd5, %rd2;
; SM70-NEXT:    selp.f64 %rd17, %rd14, %rd16, %p7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd27, [%rd9+8], %rd5, %rd17;
; SM70-NEXT:    setp.ne.b64 %p8, %rd27, %rd5;
; SM70-NEXT:    @%p8 bra $L__BB286_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end7
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd28, [%rd9+16];
; SM70-NEXT:  $L__BB286_5: // %atomicrmw.start19
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd7, %rd28;
; SM70-NEXT:    setp.eq.b64 %p9, %rd7, 0;
; SM70-NEXT:    selp.f64 %rd18, %rd7, %rd1, %p9;
; SM70-NEXT:    setp.nan.f64 %p10, %rd7, %rd1;
; SM70-NEXT:    max.f64 %rd19, %rd7, %rd1;
; SM70-NEXT:    selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p10;
; SM70-NEXT:    setp.eq.f64 %p11, %rd7, %rd1;
; SM70-NEXT:    selp.f64 %rd21, %rd18, %rd20, %p11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd28, [%rd9+16], %rd7, %rd21;
; SM70-NEXT:    setp.ne.b64 %p12, %rd28, %rd7;
; SM70-NEXT:    @%p12 bra $L__BB286_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end18
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd29, [%rd9+24];
; SM70-NEXT:  $L__BB286_7: // %atomicrmw.start26
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    setp.eq.b64 %p13, %rd29, 0;
; SM70-NEXT:    selp.f64 %rd22, %rd29, %rd6, %p13;
; SM70-NEXT:    setp.nan.f64 %p14, %rd29, %rd6;
; SM70-NEXT:    max.f64 %rd23, %rd29, %rd6;
; SM70-NEXT:    selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p14;
; SM70-NEXT:    setp.eq.f64 %p15, %rd29, %rd6;
; SM70-NEXT:    selp.f64 %rd25, %rd22, %rd24, %p15;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd8, [%rd9+24], %rd29, %rd25;
; SM70-NEXT:    setp.ne.b64 %p16, %rd8, %rd29;
; SM70-NEXT:    mov.b64 %rd29, %rd8;
; SM70-NEXT:    @%p16 bra $L__BB286_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end25
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd28, %rd8};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd26, %rd27};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <4 x double> %val syncscope("block") acq_rel
        ret <4 x double> %retval
}

define <8 x double> @fmaximum_acq_rel_v8f64_global_cta(ptr addrspace(1) %addr, <8 x double> %val) {
; SM70-LABEL: fmaximum_acq_rel_v8f64_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<33>;
; SM70-NEXT:    .reg .b64 %rd<58>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd17, [fmaximum_acq_rel_v8f64_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b64 {%rd10, %rd14}, [fmaximum_acq_rel_v8f64_global_cta_param_1+48];
; SM70-NEXT:    ld.param.v2.b64 {%rd1, %rd11}, [fmaximum_acq_rel_v8f64_global_cta_param_1+32];
; SM70-NEXT:    ld.param.v2.b64 {%rd2, %rd7}, [fmaximum_acq_rel_v8f64_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v2.b64 {%rd4, %rd3}, [fmaximum_acq_rel_v8f64_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd50, [%rd17];
; SM70-NEXT:  $L__BB287_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd5, %rd50;
; SM70-NEXT:    setp.eq.b64 %p1, %rd5, 0;
; SM70-NEXT:    selp.f64 %rd18, %rd5, %rd4, %p1;
; SM70-NEXT:    setp.nan.f64 %p2, %rd5, %rd4;
; SM70-NEXT:    max.f64 %rd19, %rd5, %rd4;
; SM70-NEXT:    selp.f64 %rd20, 0d7FF8000000000000, %rd19, %p2;
; SM70-NEXT:    setp.eq.f64 %p3, %rd5, %rd4;
; SM70-NEXT:    selp.f64 %rd21, %rd18, %rd20, %p3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd50, [%rd17], %rd5, %rd21;
; SM70-NEXT:    setp.ne.b64 %p4, %rd50, %rd5;
; SM70-NEXT:    @%p4 bra $L__BB287_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd51, [%rd17+8];
; SM70-NEXT:  $L__BB287_3: // %atomicrmw.start12
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd6, %rd51;
; SM70-NEXT:    setp.eq.b64 %p5, %rd6, 0;
; SM70-NEXT:    selp.f64 %rd22, %rd6, %rd3, %p5;
; SM70-NEXT:    setp.nan.f64 %p6, %rd6, %rd3;
; SM70-NEXT:    max.f64 %rd23, %rd6, %rd3;
; SM70-NEXT:    selp.f64 %rd24, 0d7FF8000000000000, %rd23, %p6;
; SM70-NEXT:    setp.eq.f64 %p7, %rd6, %rd3;
; SM70-NEXT:    selp.f64 %rd25, %rd22, %rd24, %p7;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd51, [%rd17+8], %rd6, %rd25;
; SM70-NEXT:    setp.ne.b64 %p8, %rd51, %rd6;
; SM70-NEXT:    @%p8 bra $L__BB287_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end11
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd52, [%rd17+16];
; SM70-NEXT:  $L__BB287_5: // %atomicrmw.start23
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd8, %rd52;
; SM70-NEXT:    setp.eq.b64 %p9, %rd8, 0;
; SM70-NEXT:    selp.f64 %rd26, %rd8, %rd2, %p9;
; SM70-NEXT:    setp.nan.f64 %p10, %rd8, %rd2;
; SM70-NEXT:    max.f64 %rd27, %rd8, %rd2;
; SM70-NEXT:    selp.f64 %rd28, 0d7FF8000000000000, %rd27, %p10;
; SM70-NEXT:    setp.eq.f64 %p11, %rd8, %rd2;
; SM70-NEXT:    selp.f64 %rd29, %rd26, %rd28, %p11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd52, [%rd17+16], %rd8, %rd29;
; SM70-NEXT:    setp.ne.b64 %p12, %rd52, %rd8;
; SM70-NEXT:    @%p12 bra $L__BB287_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end22
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd53, [%rd17+24];
; SM70-NEXT:  $L__BB287_7: // %atomicrmw.start30
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd9, %rd53;
; SM70-NEXT:    setp.eq.b64 %p13, %rd9, 0;
; SM70-NEXT:    selp.f64 %rd30, %rd9, %rd7, %p13;
; SM70-NEXT:    setp.nan.f64 %p14, %rd9, %rd7;
; SM70-NEXT:    max.f64 %rd31, %rd9, %rd7;
; SM70-NEXT:    selp.f64 %rd32, 0d7FF8000000000000, %rd31, %p14;
; SM70-NEXT:    setp.eq.f64 %p15, %rd9, %rd7;
; SM70-NEXT:    selp.f64 %rd33, %rd30, %rd32, %p15;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd53, [%rd17+24], %rd9, %rd33;
; SM70-NEXT:    setp.ne.b64 %p16, %rd53, %rd9;
; SM70-NEXT:    @%p16 bra $L__BB287_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end29
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd54, [%rd17+32];
; SM70-NEXT:  $L__BB287_9: // %atomicrmw.start45
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd12, %rd54;
; SM70-NEXT:    setp.eq.b64 %p17, %rd12, 0;
; SM70-NEXT:    selp.f64 %rd34, %rd12, %rd1, %p17;
; SM70-NEXT:    setp.nan.f64 %p18, %rd12, %rd1;
; SM70-NEXT:    max.f64 %rd35, %rd12, %rd1;
; SM70-NEXT:    selp.f64 %rd36, 0d7FF8000000000000, %rd35, %p18;
; SM70-NEXT:    setp.eq.f64 %p19, %rd12, %rd1;
; SM70-NEXT:    selp.f64 %rd37, %rd34, %rd36, %p19;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd54, [%rd17+32], %rd12, %rd37;
; SM70-NEXT:    setp.ne.b64 %p20, %rd54, %rd12;
; SM70-NEXT:    @%p20 bra $L__BB287_9;
; SM70-NEXT:  // %bb.10: // %atomicrmw.end44
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd55, [%rd17+40];
; SM70-NEXT:  $L__BB287_11: // %atomicrmw.start52
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd13, %rd55;
; SM70-NEXT:    setp.eq.b64 %p21, %rd13, 0;
; SM70-NEXT:    selp.f64 %rd38, %rd13, %rd11, %p21;
; SM70-NEXT:    setp.nan.f64 %p22, %rd13, %rd11;
; SM70-NEXT:    max.f64 %rd39, %rd13, %rd11;
; SM70-NEXT:    selp.f64 %rd40, 0d7FF8000000000000, %rd39, %p22;
; SM70-NEXT:    setp.eq.f64 %p23, %rd13, %rd11;
; SM70-NEXT:    selp.f64 %rd41, %rd38, %rd40, %p23;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd55, [%rd17+40], %rd13, %rd41;
; SM70-NEXT:    setp.ne.b64 %p24, %rd55, %rd13;
; SM70-NEXT:    @%p24 bra $L__BB287_11;
; SM70-NEXT:  // %bb.12: // %atomicrmw.end51
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd56, [%rd17+48];
; SM70-NEXT:  $L__BB287_13: // %atomicrmw.start63
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    mov.b64 %rd15, %rd56;
; SM70-NEXT:    setp.eq.b64 %p25, %rd15, 0;
; SM70-NEXT:    selp.f64 %rd42, %rd15, %rd10, %p25;
; SM70-NEXT:    setp.nan.f64 %p26, %rd15, %rd10;
; SM70-NEXT:    max.f64 %rd43, %rd15, %rd10;
; SM70-NEXT:    selp.f64 %rd44, 0d7FF8000000000000, %rd43, %p26;
; SM70-NEXT:    setp.eq.f64 %p27, %rd15, %rd10;
; SM70-NEXT:    selp.f64 %rd45, %rd42, %rd44, %p27;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd56, [%rd17+48], %rd15, %rd45;
; SM70-NEXT:    setp.ne.b64 %p28, %rd56, %rd15;
; SM70-NEXT:    @%p28 bra $L__BB287_13;
; SM70-NEXT:  // %bb.14: // %atomicrmw.end62
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd57, [%rd17+56];
; SM70-NEXT:  $L__BB287_15: // %atomicrmw.start70
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    setp.eq.b64 %p29, %rd57, 0;
; SM70-NEXT:    selp.f64 %rd46, %rd57, %rd14, %p29;
; SM70-NEXT:    setp.nan.f64 %p30, %rd57, %rd14;
; SM70-NEXT:    max.f64 %rd47, %rd57, %rd14;
; SM70-NEXT:    selp.f64 %rd48, 0d7FF8000000000000, %rd47, %p30;
; SM70-NEXT:    setp.eq.f64 %p31, %rd57, %rd14;
; SM70-NEXT:    selp.f64 %rd49, %rd46, %rd48, %p31;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd16, [%rd17+56], %rd57, %rd49;
; SM70-NEXT:    setp.ne.b64 %p32, %rd16, %rd57;
; SM70-NEXT:    mov.b64 %rd57, %rd16;
; SM70-NEXT:    @%p32 bra $L__BB287_15;
; SM70-NEXT:  // %bb.16: // %atomicrmw.end69
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd56, %rd16};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd54, %rd55};
; SM70-NEXT:    st.param.v2.b64 [func_retval0], {%rd50, %rd51};
; SM70-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd52, %rd53};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <8 x double> %val syncscope("block") acq_rel
        ret <8 x double> %retval
}

define <1 x half> @fadd_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
; SM70-LABEL: fadd_acq_rel_v1f16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b16 %rs<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v1f16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b16 %rs1, [fadd_acq_rel_v1f16_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %rs2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
        ret <1 x half> %retval
}

define <2 x half> @fadd_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-LABEL: fadd_acq_rel_v2f16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b16 %rs<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v2f16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fadd_acq_rel_v2f16_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs1;
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1+2], %rs2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs3, %rs4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
        ret <2 x half> %retval
}

define <4 x half> @fadd_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM70-LABEL: fadd_acq_rel_v4f16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b16 %rs<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v4f16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fadd_acq_rel_v4f16_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs5, [%rd1], %rs1;
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs2;
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs7, [%rd1+4], %rs3;
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+6], %rs4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b16 [func_retval0], {%rs5, %rs6, %rs7, %rs8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
        ret <4 x half> %retval
}

define <8 x half> @fadd_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM70-LABEL: fadd_acq_rel_v8f16_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b16 %rs<17>;
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v8f16_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8f16_global_cta_param_1];
; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs9, [%rd1], %rs7;
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+2], %rs8;
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs11, [%rd1+4], %rs5;
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs6;
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs13, [%rd1+8], %rs3;
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+10], %rs4;
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs15, [%rd1+12], %rs1;
; SM70-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+14], %rs2;
; SM70-NEXT:    mov.b32 %r5, {%rs9, %rs10};
; SM70-NEXT:    mov.b32 %r6, {%rs11, %rs12};
; SM70-NEXT:    mov.b32 %r7, {%rs13, %rs14};
; SM70-NEXT:    mov.b32 %r8, {%rs15, %rs16};
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
        ret <8 x half> %retval
}

define <1 x half> @fsub_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v1f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<4>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs2, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v1f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<4>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs2, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v1f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<4>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs2, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %rs3;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v1f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<4>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fsub_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs2, %rs1;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs3, [%rd1], %rs2;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %rs3;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
        ret <1 x half> %retval
}

define <2 x half> @fsub_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fsub_acq_rel_v2f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1], %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs5, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs5;
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs6};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fsub_acq_rel_v2f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1], %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs5, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs5;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs6};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fsub_acq_rel_v2f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1], %rs3;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs5, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs5;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs6};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v2f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v2f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fsub_acq_rel_v2f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs4, [%rd1], %rs3;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs5, %rs2;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1+2], %rs5;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs6};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
        ret <2 x half> %retval
}

define <4 x half> @fsub_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fsub_acq_rel_v4f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs5, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1], %rs5;
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs7, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+2], %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs9, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+4], %rs9;
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs11, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs11;
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b16 [func_retval0], {%rs6, %rs8, %rs10, %rs12};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fsub_acq_rel_v4f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs5, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1], %rs5;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs7, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+2], %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs9, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+4], %rs9;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs11, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs11;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b16 [func_retval0], {%rs6, %rs8, %rs10, %rs12};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fsub_acq_rel_v4f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs5, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1], %rs5;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs7, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+2], %rs7;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs9, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+4], %rs9;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs11, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs11;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b16 [func_retval0], {%rs6, %rs8, %rs10, %rs12};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v4f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v4f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [fsub_acq_rel_v4f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs5, %rs1;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs6, [%rd1], %rs5;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs7, %rs2;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs8, [%rd1+2], %rs7;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs9, %rs3;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1+4], %rs9;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs11, %rs4;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+6], %rs11;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b16 [func_retval0], {%rs6, %rs8, %rs10, %rs12};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
        ret <4 x half> %retval
}

define <8 x half> @fsub_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<9>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs9, %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1], %rs9;
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs11, %rs8;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+2], %rs11;
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs13, %rs5;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+4], %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs15, %rs6;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+6], %rs15;
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs17, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs18, [%rd1+8], %rs17;
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs19, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs20, [%rd1+10], %rs19;
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs21, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs22, [%rd1+12], %rs21;
; SM70-NOFTZ-DISALLOW-NEXT:    neg.f16 %rs23, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs24, [%rd1+14], %rs23;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs10, %rs12};
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r6, {%rs14, %rs16};
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r7, {%rs18, %rs20};
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r8, {%rs22, %rs24};
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<9>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs9, %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1], %rs9;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs11, %rs8;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+2], %rs11;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs13, %rs5;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+4], %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs15, %rs6;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+6], %rs15;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs17, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs18, [%rd1+8], %rs17;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs19, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs20, [%rd1+10], %rs19;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs21, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs22, [%rd1+12], %rs21;
; SM70-NOFTZ-ALLOW-NEXT:    neg.f16 %rs23, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs24, [%rd1+14], %rs23;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs10, %rs12};
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r6, {%rs14, %rs16};
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r7, {%rs18, %rs20};
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r8, {%rs22, %rs24};
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<9>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs9, %rs7;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1], %rs9;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs11, %rs8;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+2], %rs11;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs13, %rs5;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+4], %rs13;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs15, %rs6;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+6], %rs15;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs17, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs18, [%rd1+8], %rs17;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs19, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs20, [%rd1+10], %rs19;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs21, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs22, [%rd1+12], %rs21;
; SM70-FTZ-DISALLOW-NEXT:    neg.ftz.f16 %rs23, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs24, [%rd1+14], %rs23;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs10, %rs12};
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r6, {%rs14, %rs16};
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r7, {%rs18, %rs20};
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r8, {%rs22, %rs24};
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v8f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<9>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v8f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r1;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs9, %rs7;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs10, [%rd1], %rs9;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs11, %rs8;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs12, [%rd1+2], %rs11;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs13, %rs5;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs14, [%rd1+4], %rs13;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs15, %rs6;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs16, [%rd1+6], %rs15;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs17, %rs3;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs18, [%rd1+8], %rs17;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs19, %rs4;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs20, [%rd1+10], %rs19;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs21, %rs1;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs22, [%rd1+12], %rs21;
; SM70-FTZ-ALLOW-NEXT:    neg.ftz.f16 %rs23, %rs2;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.add.noftz.f16 %rs24, [%rd1+14], %rs23;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs10, %rs12};
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r6, {%rs14, %rs16};
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r7, {%rs18, %rs20};
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r8, {%rs22, %rs24};
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
        ret <8 x half> %retval
}

define <1 x half> @fmin_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v1f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<4>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB296_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r10, %r8, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r3, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB296_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v1f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<4>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB296_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r10, %r8, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p1, %r3, %r16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB296_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_v1f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<4>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB296_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r10, %r8, %r9;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r3, %r16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB296_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_v1f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<4>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fmin_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fmin_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
; SM70-FTZ-ALLOW-NEXT:  $L__BB296_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r10, %r8, %r9;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p1, %r3, %r16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB296_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
        ret <1 x half> %retval
}

define <2 x half> @fmin_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<11>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB297_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r5, %r4, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r8, %r7, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r1, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r10, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB297_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<11>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB297_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r5, %r4, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r8, %r7, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p1, %r1, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r10, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB297_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_v2f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<11>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB297_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r5, %r4, %r3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r8, %r7, %r6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r1, %r10;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r10, %r1;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB297_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_v2f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<11>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fmin_acq_rel_v2f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v2f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
; SM70-FTZ-ALLOW-NEXT:  $L__BB297_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r5, %r4, %r3;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r8, %r7, %r6;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p1, %r1, %r10;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r10, %r1;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB297_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
        ret <2 x half> %retval
}

define <4 x half> @fmin_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<19>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB298_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r5, %r4, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r8, %r7, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r11, %rs10;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r12, %r11, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r13, %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r14, %rs9;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r15, %r14, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r16, {%rs12, %rs11};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB298_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r17, %r18};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<19>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB298_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r5, %r4, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r8, %r7, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r11, %rs10;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r12, %r11, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r13, %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r14, %rs9;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r15, %r14, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r16, {%rs12, %rs11};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB298_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r17, %r18};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_v4f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<19>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB298_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r5, %r4, %r3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r8, %r7, %r6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r11, %rs10;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r12, %r11, %r10;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r12;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs7;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r14, %rs9;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r15, %r14, %r13;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r15;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r16, {%rs12, %rs11};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB298_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r17, %r18};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_v4f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<19>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v4f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
; SM70-FTZ-ALLOW-NEXT:  $L__BB298_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r5, %r4, %r3;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r8, %r7, %r6;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r11, %rs10;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r12, %r11, %r10;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r12;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs7;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r14, %rs9;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r15, %r14, %r13;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r15;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r16, {%rs12, %rs11};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB298_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r17, %r18};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
        ret <4 x half> %retval
}

define <8 x half> @fmin_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<37>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB299_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r5, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r33;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r7, %r6, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r8, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r10, %r9, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r12, %rs8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r13, %rs10;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r14, %r13, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r14;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r15, %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r16, %rs9;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r17, %r16, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r18, {%rs12, %rs11};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB299_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB299_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r19, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r35;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r20, %rs16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r21, %r20, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs17, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r22, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r23, %rs15;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r24, %r23, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r26, %rs20;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r27, %rs22;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r28, %r27, %r26;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs23, %r28;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r29, %rs19;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r30, %rs21;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r31, %r30, %r29;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs24, %r31;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r32, {%rs24, %rs23};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r32;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r35;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r36;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB299_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<37>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB299_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r5, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r33;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r7, %r6, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r8, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r10, %r9, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r12, %rs8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r13, %rs10;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r14, %r13, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r14;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r15, %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r16, %rs9;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r17, %r16, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r18, {%rs12, %rs11};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB299_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB299_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r19, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r35;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r20, %rs16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r21, %r20, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs17, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r22, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r23, %rs15;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r24, %r23, %r22;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r26, %rs20;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r27, %rs22;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r28, %r27, %r26;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs23, %r28;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r29, %rs19;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r30, %rs21;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r31, %r30, %r29;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs24, %r31;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r32, {%rs24, %rs23};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r32;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r35;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r36;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p2 bra $L__BB299_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_v8f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<37>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB299_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r5, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r33;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r7, %r6, %r5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r7;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r10, %r9, %r8;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r12, %rs8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs10;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r14, %r13, %r12;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r14;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r15, %rs7;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r16, %rs9;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r17, %r16, %r15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r17;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r18, {%rs12, %rs11};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r18;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB299_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB299_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r19, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r35;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r20, %rs16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r21, %r20, %r19;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs17, %r21;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r22, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r23, %rs15;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r24, %r23, %r22;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r26, %rs20;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r27, %rs22;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r28, %r27, %r26;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs23, %r28;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r29, %rs19;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r30, %rs21;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r31, %r30, %r29;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs24, %r31;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r32, {%rs24, %rs23};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r32;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r35;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r36;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p2 bra $L__BB299_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_v8f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<37>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v8f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB299_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r5, %rs2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r33;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs4;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r7, %r6, %r5;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r7;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs1;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs3;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r10, %r9, %r8;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r12, %rs8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs10;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r14, %r13, %r12;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r14;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r15, %rs7;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r16, %rs9;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r17, %r16, %r15;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r17;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r18, {%rs12, %rs11};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r18;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB299_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-FTZ-ALLOW-NEXT:  $L__BB299_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r19, %rs14;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r35;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r20, %rs16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r21, %r20, %r19;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs17, %r21;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r22, %rs13;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r23, %rs15;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r24, %r23, %r22;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r26, %rs20;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r27, %rs22;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r28, %r27, %r26;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs23, %r28;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r29, %rs19;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r30, %rs21;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r31, %r30, %r29;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs24, %r31;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r32, {%rs24, %rs23};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r32;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r35;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r36;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p2 bra $L__BB299_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
        ret <8 x half> %retval
}

define <1 x half> @fmax_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<4>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB300_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r10, %r8, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r3, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB300_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<4>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB300_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r8, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r10, %r8, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p1, %r3, %r16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB300_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<4>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB300_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r10, %r8, %r9;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r3, %r16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB300_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_v1f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<4>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fmax_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fmax_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs1;
; SM70-FTZ-ALLOW-NEXT:  $L__BB300_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs2;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r10, %r8, %r9;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs3, %r10;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p1, %r3, %r16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB300_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
        ret <1 x half> %retval
}

define <2 x half> @fmax_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<11>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB301_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r5, %r4, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r8, %r7, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r1, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r10, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB301_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<11>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB301_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r5, %r4, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r8, %r7, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p1, %r1, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r10, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB301_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_v2f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<11>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB301_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r5, %r4, %r3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r8, %r7, %r6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p1, %r1, %r10;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r10, %r1;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB301_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_v2f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<11>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fmax_acq_rel_v2f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v2f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r10, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
; SM70-FTZ-ALLOW-NEXT:  $L__BB301_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r10;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r5, %r4, %r3;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r8, %r7, %r6;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r10, %r9;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p1, %r1, %r10;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r10, %r1;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB301_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
        ret <2 x half> %retval
}

define <4 x half> @fmax_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<19>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB302_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r5, %r4, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r8, %r7, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r11, %rs10;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r12, %r11, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r13, %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r14, %rs9;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r15, %r14, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r16, {%rs12, %rs11};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB302_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r17, %r18};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<19>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r10, %rs8;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB302_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r4, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r5, %r4, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r7, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r8, %r7, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r11, %rs10;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r12, %r11, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r13, %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r14, %rs9;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r15, %r14, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r16, {%rs12, %rs11};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB302_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r17, %r18};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_v4f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<19>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB302_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r5, %r4, %r3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r8, %r7, %r6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r11, %rs10;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r12, %r11, %r10;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r12;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs7;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r14, %rs9;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r15, %r14, %r13;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r15;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r16, {%rs12, %rs11};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB302_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r17, %r18};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_v4f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<19>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v4f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r10, %rs8;
; SM70-FTZ-ALLOW-NEXT:  $L__BB302_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r17;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r4, %rs4;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r5, %r4, %r3;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs1;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r7, %rs3;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r8, %r7, %r6;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r9, {%rs6, %rs5};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r18;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r11, %rs10;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r12, %r11, %r10;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r12;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs7;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r14, %rs9;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r15, %r14, %r13;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r15;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r16, {%rs12, %rs11};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r16;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r17;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r18;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r17, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB302_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r17, %r18};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
        ret <4 x half> %retval
}

define <8 x half> @fmax_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<37>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB303_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r5, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r33;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r6, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r7, %r6, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r8, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r9, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r10, %r9, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r12, %rs8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r13, %rs10;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r14, %r13, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r14;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r15, %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r16, %rs9;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r17, %r16, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r18, {%rs12, %rs11};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p1 bra $L__BB303_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB303_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r19, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r35;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r20, %rs16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r21, %r20, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs17, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r22, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r23, %rs15;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r24, %r23, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r26, %rs20;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r27, %rs22;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r28, %r27, %r26;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs23, %r28;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r29, %rs19;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.f32.f16 %r30, %rs21;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r31, %r30, %r29;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs24, %r31;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r32, {%rs24, %rs23};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r32;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r35;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r36;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB303_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<37>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB303_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r5, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r33;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r6, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r7, %r6, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r8, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r9, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r10, %r9, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r12, %rs8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r13, %rs10;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r14, %r13, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r14;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r15, %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r16, %rs9;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r17, %r16, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r18, {%rs12, %rs11};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p1 bra $L__BB303_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB303_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r19, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r35;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r20, %rs16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r21, %r20, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs17, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r22, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r23, %rs15;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r24, %r23, %r22;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r26, %rs20;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r27, %rs22;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r28, %r27, %r26;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs23, %r28;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r29, %rs19;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.f32.f16 %r30, %rs21;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r31, %r30, %r29;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs24, %r31;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r32, {%rs24, %rs23};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r32;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r35;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r36;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p2 bra $L__BB303_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_v8f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<37>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB303_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r5, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r33;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r7, %r6, %r5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r7;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r10, %r9, %r8;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r12, %rs8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs10;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r14, %r13, %r12;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r14;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r15, %rs7;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r16, %rs9;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r17, %r16, %r15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r17;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r18, {%rs12, %rs11};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r18;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p1 bra $L__BB303_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB303_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r19, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r35;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r20, %rs16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r21, %r20, %r19;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs17, %r21;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r22, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r23, %rs15;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r24, %r23, %r22;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r26, %rs20;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r27, %rs22;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r28, %r27, %r26;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs23, %r28;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r29, %rs19;
; SM70-FTZ-DISALLOW-NEXT:    cvt.ftz.f32.f16 %r30, %rs21;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r31, %r30, %r29;
; SM70-FTZ-DISALLOW-NEXT:    cvt.rn.f16.f32 %rs24, %r31;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r32, {%rs24, %rs23};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r32;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r35;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r36;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r35, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p2 bra $L__BB303_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_v8f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<37>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v8f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB303_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r5, %rs2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r33;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r6, %rs4;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r7, %r6, %r5;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs5, %r7;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r8, %rs1;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r9, %rs3;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r10, %r9, %r8;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs6, %r10;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs6, %rs5};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r12, %rs8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r34;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r13, %rs10;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r14, %r13, %r12;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs11, %r14;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r15, %rs7;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r16, %rs9;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r17, %r16, %r15;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs12, %r17;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r18, {%rs12, %rs11};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r18;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p1 bra $L__BB303_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd12;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r3;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs19, %rs20}, %r4;
; SM70-FTZ-ALLOW-NEXT:  $L__BB303_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r19, %rs14;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r35;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r20, %rs16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r21, %r20, %r19;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs17, %r21;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r22, %rs13;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r23, %rs15;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r24, %r23, %r22;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs18, %r24;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r25, {%rs18, %rs17};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r25;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r26, %rs20;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs21, %rs22}, %r36;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r27, %rs22;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r28, %r27, %r26;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs23, %r28;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r29, %rs19;
; SM70-FTZ-ALLOW-NEXT:    cvt.ftz.f32.f16 %r30, %rs21;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r31, %r30, %r29;
; SM70-FTZ-ALLOW-NEXT:    cvt.rn.f16.f32 %rs24, %r31;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r32, {%rs24, %rs23};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r32;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r35;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r36;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p2, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r36}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r35, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p2 bra $L__BB303_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r33, %r34, %r35, %r36};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
        ret <8 x half> %retval
}

define <1 x half> @fminimum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_v1f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<14>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB304_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p2, %rs2, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p3, %rs2, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, 0x7E00, %rs4, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p4, %rs2, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r13, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p5 bra $L__BB304_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v1f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<14>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB304_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p2, %rs2, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p3, %rs2, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, 0x7E00, %rs4, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p4, %rs2, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r13, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    @%p5 bra $L__BB304_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v1f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<14>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:  $L__BB304_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p2, %rs2, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs2, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, 0x7E00, %rs4, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs2, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r13;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r13, %r3;
; SM70-FTZ-DISALLOW-NEXT:    @%p5 bra $L__BB304_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_v1f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<14>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fminimum_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fminimum_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-FTZ-ALLOW-NEXT:  $L__BB304_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p2, %rs2, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs2, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, 0x7E00, %rs4, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs2, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r13;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r13, %r3;
; SM70-FTZ-ALLOW-NEXT:    @%p5 bra $L__BB304_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
        ret <1 x half> %retval
}

define <2 x half> @fminimum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<10>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fminimum_acq_rel_v2f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v2f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB305_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p2, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p3, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p4, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p6, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p7, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p8, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r4, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p9 bra $L__BB305_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<10>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fminimum_acq_rel_v2f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v2f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB305_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p2, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p3, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p4, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p6, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p7, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p8, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r4, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    @%p9 bra $L__BB305_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v2f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<10>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fminimum_acq_rel_v2f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v2f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB305_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p2, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p6, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p7, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p8, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r4;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r4, %r1;
; SM70-FTZ-DISALLOW-NEXT:    @%p9 bra $L__BB305_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_v2f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<10>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fminimum_acq_rel_v2f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v2f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB305_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p2, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p6, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p7, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p8, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r4;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r4, %r1;
; SM70-FTZ-ALLOW-NEXT:    @%p9 bra $L__BB305_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
        ret <2 x half> %retval
}

define <4 x half> @fminimum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<18>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v4f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB306_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p2, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p3, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p4, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p6, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p7, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p8, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p10, %rs16, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p11, %rs16, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p12, %rs16, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p14, %rs15, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p15, %rs15, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p16, %rs15, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r4, {%rs24, %rs20};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p17 bra $L__BB306_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<18>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v4f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB306_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p2, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p3, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p4, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p6, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p7, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p8, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p10, %rs16, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p11, %rs16, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p12, %rs16, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p14, %rs15, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p15, %rs15, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p16, %rs15, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r4, {%rs24, %rs20};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p17 bra $L__BB306_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v4f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<18>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v4f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB306_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p2, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p6, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p7, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p8, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p10, %rs16, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p11, %rs16, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p12, %rs16, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p14, %rs15, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p15, %rs15, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p16, %rs15, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r4, {%rs24, %rs20};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p17 bra $L__BB306_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_v4f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<18>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v4f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB306_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p2, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p6, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p7, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p8, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p10, %rs16, %rs14;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p11, %rs16, %rs14;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p12, %rs16, %rs14;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p14, %rs15, %rs13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p15, %rs15, %rs13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p16, %rs15, %rs13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r4, {%rs24, %rs20};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p17 bra $L__BB306_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
        ret <4 x half> %retval
}

define <8 x half> @fminimum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<35>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v8f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB307_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p2, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p3, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p4, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p6, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p7, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p8, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p10, %rs16, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p11, %rs16, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p12, %rs16, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p14, %rs15, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p15, %rs15, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p16, %rs15, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r6, {%rs24, %rs20};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p17 bra $L__BB307_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB307_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p18, %rs28, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p19, %rs28, %rs26;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p20, %rs28, %rs26;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs31, 0x7E00, %rs30, %p20;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p21, %rs28, %rs26;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p22, %rs27, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p23, %rs27, %rs25;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p24, %rs27, %rs25;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs35, 0x7E00, %rs34, %p24;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p25, %rs27, %rs25;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r7, {%rs36, %rs32};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p26, %rs40, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p27, %rs40, %rs38;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p28, %rs40, %rs38;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs43, 0x7E00, %rs42, %p28;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p29, %rs40, %rs38;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p30, %rs39, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f16 %p31, %rs39, %rs37;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p32, %rs39, %rs37;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs47, 0x7E00, %rs46, %p32;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p33, %rs39, %rs37;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r8, {%rs48, %rs44};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p34 bra $L__BB307_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<35>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v8f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB307_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p2, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p3, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p4, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p6, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p7, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p8, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p10, %rs16, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p11, %rs16, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p12, %rs16, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p14, %rs15, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p15, %rs15, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p16, %rs15, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r6, {%rs24, %rs20};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p17 bra $L__BB307_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB307_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p18, %rs28, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p19, %rs28, %rs26;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p20, %rs28, %rs26;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs31, 0x7E00, %rs30, %p20;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p21, %rs28, %rs26;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p22, %rs27, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p23, %rs27, %rs25;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p24, %rs27, %rs25;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs35, 0x7E00, %rs34, %p24;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p25, %rs27, %rs25;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r7, {%rs36, %rs32};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p26, %rs40, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p27, %rs40, %rs38;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p28, %rs40, %rs38;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs43, 0x7E00, %rs42, %p28;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p29, %rs40, %rs38;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p30, %rs39, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f16 %p31, %rs39, %rs37;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p32, %rs39, %rs37;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs47, 0x7E00, %rs46, %p32;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p33, %rs39, %rs37;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r8, {%rs48, %rs44};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p34 bra $L__BB307_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v8f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<35>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v8f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB307_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p2, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p6, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p7, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p8, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p10, %rs16, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p11, %rs16, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p12, %rs16, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p14, %rs15, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p15, %rs15, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p16, %rs15, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r6, {%rs24, %rs20};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p17 bra $L__BB307_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB307_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r11;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p18, %rs28, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p19, %rs28, %rs26;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p20, %rs28, %rs26;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs31, 0x7E00, %rs30, %p20;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p21, %rs28, %rs26;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p22, %rs27, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p23, %rs27, %rs25;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p24, %rs27, %rs25;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs35, 0x7E00, %rs34, %p24;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p25, %rs27, %rs25;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r7, {%rs36, %rs32};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p26, %rs40, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p27, %rs40, %rs38;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p28, %rs40, %rs38;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs43, 0x7E00, %rs42, %p28;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p29, %rs40, %rs38;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p30, %rs39, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f16 %p31, %rs39, %rs37;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p32, %rs39, %rs37;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs47, 0x7E00, %rs46, %p32;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p33, %rs39, %rs37;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r8, {%rs48, %rs44};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p34 bra $L__BB307_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_v8f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<35>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v8f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB307_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p2, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p6, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p7, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p8, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p10, %rs16, %rs14;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p11, %rs16, %rs14;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p12, %rs16, %rs14;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p14, %rs15, %rs13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p15, %rs15, %rs13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p16, %rs15, %rs13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r6, {%rs24, %rs20};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p17 bra $L__BB307_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-FTZ-ALLOW-NEXT:  $L__BB307_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r11;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p18, %rs28, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p19, %rs28, %rs26;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p20, %rs28, %rs26;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs31, 0x7E00, %rs30, %p20;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p21, %rs28, %rs26;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p22, %rs27, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p23, %rs27, %rs25;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p24, %rs27, %rs25;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs35, 0x7E00, %rs34, %p24;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p25, %rs27, %rs25;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r7, {%rs36, %rs32};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p26, %rs40, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p27, %rs40, %rs38;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p28, %rs40, %rs38;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs43, 0x7E00, %rs42, %p28;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p29, %rs40, %rs38;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p30, %rs39, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f16 %p31, %rs39, %rs37;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p32, %rs39, %rs37;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs47, 0x7E00, %rs46, %p32;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p33, %rs39, %rs37;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r8, {%rs48, %rs44};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p34 bra $L__BB307_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
        ret <8 x half> %retval
}

define <1 x half> @fmaximum_acq_rel_v1f16_global_cta(ptr addrspace(1) %addr, <1 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_v1f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<14>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB308_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p2, %rs2, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p3, %rs2, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, 0x7E00, %rs4, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p4, %rs2, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r13, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p5 bra $L__BB308_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v1f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<14>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB308_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p2, %rs2, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p3, %rs2, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, 0x7E00, %rs4, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p4, %rs2, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r13, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    @%p5 bra $L__BB308_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v1f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<14>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:  $L__BB308_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p2, %rs2, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs2, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, 0x7E00, %rs4, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs2, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r13;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r13, %r3;
; SM70-FTZ-DISALLOW-NEXT:    @%p5 bra $L__BB308_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v1f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<14>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fmaximum_acq_rel_v1f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fmaximum_acq_rel_v1f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r13, [%rd1];
; SM70-FTZ-ALLOW-NEXT:  $L__BB308_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r13, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p2, %rs2, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs2, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, 0x7E00, %rs4, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs2, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r9, %r8, %r1;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r10, %r13, %r2;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r11, %r10, %r9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r13, %r11;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r13;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r13, %r3;
; SM70-FTZ-ALLOW-NEXT:    @%p5 bra $L__BB308_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r12;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <1 x half> %val syncscope("block") acq_rel
        ret <1 x half> %retval
}

define <2 x half> @fmaximum_acq_rel_v2f16_global_cta(ptr addrspace(1) %addr, <2 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<10>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fmaximum_acq_rel_v2f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v2f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB309_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p2, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p3, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p4, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p6, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p7, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p8, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r4, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p9 bra $L__BB309_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v2f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<10>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fmaximum_acq_rel_v2f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v2f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB309_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p2, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p3, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p4, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p6, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p7, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p8, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r4, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    @%p9 bra $L__BB309_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v2f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<10>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fmaximum_acq_rel_v2f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v2f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB309_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p2, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p6, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p7, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p8, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r4;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r4, %r1;
; SM70-FTZ-DISALLOW-NEXT:    @%p9 bra $L__BB309_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v2f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<10>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<5>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fmaximum_acq_rel_v2f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v2f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r4, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB309_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p2, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p6, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p7, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p8, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r4;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r4, %r1;
; SM70-FTZ-ALLOW-NEXT:    @%p9 bra $L__BB309_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <2 x half> %val syncscope("block") acq_rel
        ret <2 x half> %retval
}

define <4 x half> @fmaximum_acq_rel_v4f16_global_cta(ptr addrspace(1) %addr, <4 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<18>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v4f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB310_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p2, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p3, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p4, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p6, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p7, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p8, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p10, %rs16, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p11, %rs16, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p12, %rs16, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p14, %rs15, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p15, %rs15, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p16, %rs15, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r4, {%rs24, %rs20};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p17 bra $L__BB310_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v4f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<18>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v4f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB310_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p2, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p3, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p4, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p6, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p7, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p8, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p10, %rs16, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p11, %rs16, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p12, %rs16, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p14, %rs15, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p15, %rs15, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p16, %rs15, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r4, {%rs24, %rs20};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p17 bra $L__BB310_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v4f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<18>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v4f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB310_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p2, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p6, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p7, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p8, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p10, %rs16, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p11, %rs16, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p12, %rs16, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p14, %rs15, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p15, %rs15, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p16, %rs15, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r4, {%rs24, %rs20};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p17 bra $L__BB310_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v4f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<18>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<7>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v4f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB310_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p2, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p6, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p7, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p8, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r3, {%rs12, %rs8};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r3;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r6;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p10, %rs16, %rs14;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p11, %rs16, %rs14;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p12, %rs16, %rs14;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p14, %rs15, %rs13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p15, %rs15, %rs13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p16, %rs15, %rs13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r4, {%rs24, %rs20};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r4;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r5, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p17 bra $L__BB310_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <4 x half> %val syncscope("block") acq_rel
        ret <4 x half> %retval
}

define <8 x half> @fmaximum_acq_rel_v8f16_global_cta(ptr addrspace(1) %addr, <8 x half> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<35>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v8f16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB311_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p2, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p3, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p4, %rs4, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p6, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p7, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p8, %rs3, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p10, %rs16, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p11, %rs16, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p12, %rs16, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p14, %rs15, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p15, %rs15, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p16, %rs15, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r6, {%rs24, %rs20};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p17 bra $L__BB311_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB311_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p18, %rs28, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p19, %rs28, %rs26;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p20, %rs28, %rs26;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs31, 0x7E00, %rs30, %p20;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p21, %rs28, %rs26;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p22, %rs27, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p23, %rs27, %rs25;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p24, %rs27, %rs25;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs35, 0x7E00, %rs34, %p24;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p25, %rs27, %rs25;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r7, {%rs36, %rs32};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p26, %rs40, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p27, %rs40, %rs38;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p28, %rs40, %rs38;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs43, 0x7E00, %rs42, %p28;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p29, %rs40, %rs38;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p30, %rs39, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f16 %p31, %rs39, %rs37;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f16 %p32, %rs39, %rs37;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs47, 0x7E00, %rs46, %p32;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f16 %p33, %rs39, %rs37;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r8, {%rs48, %rs44};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p34 bra $L__BB311_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v8f16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<35>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v8f16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB311_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p2, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p3, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p4, %rs4, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p6, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p7, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p8, %rs3, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p10, %rs16, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p11, %rs16, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p12, %rs16, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p14, %rs15, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p15, %rs15, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p16, %rs15, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r6, {%rs24, %rs20};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p17 bra $L__BB311_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB311_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p18, %rs28, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p19, %rs28, %rs26;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p20, %rs28, %rs26;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs31, 0x7E00, %rs30, %p20;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p21, %rs28, %rs26;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p22, %rs27, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p23, %rs27, %rs25;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p24, %rs27, %rs25;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs35, 0x7E00, %rs34, %p24;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p25, %rs27, %rs25;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r7, {%rs36, %rs32};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p26, %rs40, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p27, %rs40, %rs38;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p28, %rs40, %rs38;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs43, 0x7E00, %rs42, %p28;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p29, %rs40, %rs38;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p30, %rs39, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f16 %p31, %rs39, %rs37;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f16 %p32, %rs39, %rs37;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs47, 0x7E00, %rs46, %p32;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f16 %p33, %rs39, %rs37;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r8, {%rs48, %rs44};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p34 bra $L__BB311_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v8f16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<35>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v8f16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB311_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p2, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs4, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p6, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p7, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p8, %rs3, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p10, %rs16, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p11, %rs16, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p12, %rs16, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p14, %rs15, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p15, %rs15, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p16, %rs15, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r6, {%rs24, %rs20};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p17 bra $L__BB311_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB311_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r11;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p18, %rs28, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p19, %rs28, %rs26;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p20, %rs28, %rs26;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs31, 0x7E00, %rs30, %p20;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p21, %rs28, %rs26;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p22, %rs27, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p23, %rs27, %rs25;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p24, %rs27, %rs25;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs35, 0x7E00, %rs34, %p24;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p25, %rs27, %rs25;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r7, {%rs36, %rs32};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p26, %rs40, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p27, %rs40, %rs38;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p28, %rs40, %rs38;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs43, 0x7E00, %rs42, %p28;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p29, %rs40, %rs38;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p30, %rs39, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f16 %p31, %rs39, %rs37;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f16 %p32, %rs39, %rs37;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs47, 0x7E00, %rs46, %p32;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f16 %p33, %rs39, %rs37;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r8, {%rs48, %rs44};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p34 bra $L__BB311_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v8f16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<35>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v8f16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8f16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB311_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r9;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p2, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p3, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7E00, %rs6, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p4, %rs4, %rs2;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p6, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p7, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p8, %rs3, %rs1;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r5, {%rs12, %rs8};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r10;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p10, %rs16, %rs14;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p11, %rs16, %rs14;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7E00, %rs18, %p11;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p12, %rs16, %rs14;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p14, %rs15, %rs13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p15, %rs15, %rs13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7E00, %rs22, %p15;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p16, %rs15, %rs13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r6, {%rs24, %rs20};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r9;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r10;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r9, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p17 bra $L__BB311_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd12;
; SM70-FTZ-ALLOW-NEXT:  $L__BB311_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r11;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p18, %rs28, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p19, %rs28, %rs26;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p20, %rs28, %rs26;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs31, 0x7E00, %rs30, %p20;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p21, %rs28, %rs26;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p22, %rs27, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p23, %rs27, %rs25;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p24, %rs27, %rs25;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs35, 0x7E00, %rs34, %p24;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p25, %rs27, %rs25;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r7, {%rs36, %rs32};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r7;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p26, %rs40, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p27, %rs40, %rs38;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p28, %rs40, %rs38;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs43, 0x7E00, %rs42, %p28;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p29, %rs40, %rs38;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p30, %rs39, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f16 %p31, %rs39, %rs37;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f16 %p32, %rs39, %rs37;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs47, 0x7E00, %rs46, %p32;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f16 %p33, %rs39, %rs37;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r8, {%rs48, %rs44};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r8;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r11;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r11, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p34 bra $L__BB311_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <8 x half> %val syncscope("block") acq_rel
        ret <8 x half> %retval
}

define <1 x bfloat> @fadd_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_v1bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB312_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r11, %r8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r11, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB312_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_v1bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB312_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r11, %r8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r11, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    @%p2 bra $L__BB312_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_v1bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB312_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r11, %r8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r11, %r11;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-FTZ-DISALLOW-NEXT:    @%p2 bra $L__BB312_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_v1bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fadd_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fadd_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB312_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r11, %r8, %r10;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r11, %r11;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-FTZ-ALLOW-NEXT:    @%p2 bra $L__BB312_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
        ret <1 x bfloat> %retval
}

define <2 x bfloat> @fadd_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_v2bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<4>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fadd_acq_rel_v2bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB313_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r7, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r18, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p3 bra $L__BB313_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_v2bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<4>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fadd_acq_rel_v2bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB313_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r7, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r18, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    @%p3 bra $L__BB313_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_v2bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<4>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fadd_acq_rel_v2bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB313_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-FTZ-DISALLOW-NEXT:    @%p3 bra $L__BB313_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_v2bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<4>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fadd_acq_rel_v2bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v2bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB313_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-FTZ-ALLOW-NEXT:    @%p3 bra $L__BB313_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
        ret <2 x bfloat> %retval
}

define <4 x bfloat> @fadd_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_v4bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB314_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r7, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r18, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r30, %r29, %r27;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r30, %r30;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r41, %r40, %r38;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p4, %r41, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p5 bra $L__BB314_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_v4bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB314_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r7, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r18, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r30, %r29, %r27;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r30, %r30;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r41, %r40, %r38;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p4, %r41, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p5 bra $L__BB314_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_v4bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB314_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r30, %r29, %r27;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r30, %r30;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r41, %r40, %r38;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r41, %r41;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p5 bra $L__BB314_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_v4bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fadd_acq_rel_v4bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v4bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB314_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r30, %r29, %r27;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r30, %r30;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r41, %r40, %r38;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r41, %r41;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p5 bra $L__BB314_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
        ret <4 x bfloat> %retval
}

define <8 x bfloat> @fadd_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fadd_acq_rel_v8bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<11>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB315_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r9, %r8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r9, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r20, %r19, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r20, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r32, %r31, %r29;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r32, %r32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r43, %r42, %r40;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p4, %r43, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p5 bra $L__BB315_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB315_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r55, %r54, %r52;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p6, %r55, %r55;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r66, %r65, %r63;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p7, %r66, %r66;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r78, %r77, %r75;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p8, %r78, %r78;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    add.rn.f32 %r89, %r88, %r86;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p9, %r89, %r89;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p10 bra $L__BB315_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fadd_acq_rel_v8bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<11>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB315_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r9, %r8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r9, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r20, %r19, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r20, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r32, %r31, %r29;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r32, %r32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r43, %r42, %r40;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p4, %r43, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p5 bra $L__BB315_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB315_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r55, %r54, %r52;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p6, %r55, %r55;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r66, %r65, %r63;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p7, %r66, %r66;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r78, %r77, %r75;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p8, %r78, %r78;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-NOFTZ-ALLOW-NEXT:    add.rn.f32 %r89, %r88, %r86;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p9, %r89, %r89;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p10 bra $L__BB315_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fadd_acq_rel_v8bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<11>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB315_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r9, %r8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r9, %r9;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r20, %r19, %r17;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r20, %r20;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r32, %r31, %r29;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r32, %r32;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r43, %r42, %r40;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r43, %r43;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p5 bra $L__BB315_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB315_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r55, %r54, %r52;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p6, %r55, %r55;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r66, %r65, %r63;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r66, %r66;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r78, %r77, %r75;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p8, %r78, %r78;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-FTZ-DISALLOW-NEXT:    add.rn.ftz.f32 %r89, %r88, %r86;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r89, %r89;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p10 bra $L__BB315_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fadd_acq_rel_v8bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<11>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fadd_acq_rel_v8bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fadd_acq_rel_v8bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB315_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r9, %r8, %r6;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r9, %r9;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r20, %r19, %r17;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r20, %r20;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r32, %r31, %r29;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r32, %r32;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r43, %r42, %r40;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r43, %r43;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p5 bra $L__BB315_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-FTZ-ALLOW-NEXT:  $L__BB315_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r55, %r54, %r52;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p6, %r55, %r55;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r66, %r65, %r63;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r66, %r66;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r78, %r77, %r75;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p8, %r78, %r78;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-FTZ-ALLOW-NEXT:    add.rn.ftz.f32 %r89, %r88, %r86;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r89, %r89;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p10 bra $L__BB315_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fadd ptr  addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
        ret <8 x bfloat> %retval
}

define <1 x bfloat> @fsub_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v1bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB316_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r11, %r8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r11, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB316_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v1bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB316_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r11, %r8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r11, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    @%p2 bra $L__BB316_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v1bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB316_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r11, %r8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r11, %r11;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-FTZ-DISALLOW-NEXT:    @%p2 bra $L__BB316_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v1bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fsub_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fsub_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB316_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r11, %r8, %r10;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r11, %r11;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-FTZ-ALLOW-NEXT:    @%p2 bra $L__BB316_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
        ret <1 x bfloat> %retval
}

define <2 x bfloat> @fsub_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v2bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<4>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fsub_acq_rel_v2bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB317_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r7, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r18, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p3 bra $L__BB317_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v2bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<4>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fsub_acq_rel_v2bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB317_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r7, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r18, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    @%p3 bra $L__BB317_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v2bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<4>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fsub_acq_rel_v2bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB317_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-FTZ-DISALLOW-NEXT:    @%p3 bra $L__BB317_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v2bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<4>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fsub_acq_rel_v2bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v2bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB317_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-FTZ-ALLOW-NEXT:    @%p3 bra $L__BB317_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
        ret <2 x bfloat> %retval
}

define <4 x bfloat> @fsub_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v4bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v4bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB318_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r7, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r18, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r30, %r29, %r27;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r30, %r30;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r41, %r40, %r38;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p4, %r41, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p5 bra $L__BB318_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v4bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v4bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB318_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r7, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r18, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r30, %r29, %r27;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r30, %r30;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r41, %r40, %r38;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p4, %r41, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p5 bra $L__BB318_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v4bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v4bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB318_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r30, %r29, %r27;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r30, %r30;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r41, %r40, %r38;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r41, %r41;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p5 bra $L__BB318_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v4bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fsub_acq_rel_v4bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v4bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB318_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r30, %r29, %r27;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r30, %r30;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r41, %r40, %r38;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r41, %r41;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p5 bra $L__BB318_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
        ret <4 x bfloat> %retval
}

define <8 x bfloat> @fsub_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fsub_acq_rel_v8bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<11>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB319_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r9, %r8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r9, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r20, %r19, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r20, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r32, %r31, %r29;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r32, %r32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r43, %r42, %r40;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p4, %r43, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p5 bra $L__BB319_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB319_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r55, %r54, %r52;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p6, %r55, %r55;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r66, %r65, %r63;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p7, %r66, %r66;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r78, %r77, %r75;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p8, %r78, %r78;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    sub.rn.f32 %r89, %r88, %r86;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p9, %r89, %r89;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p10 bra $L__BB319_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fsub_acq_rel_v8bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<11>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB319_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r9, %r8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r9, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r20, %r19, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r20, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r32, %r31, %r29;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r32, %r32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r43, %r42, %r40;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p4, %r43, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p5 bra $L__BB319_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB319_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r55, %r54, %r52;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p6, %r55, %r55;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r66, %r65, %r63;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p7, %r66, %r66;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r78, %r77, %r75;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p8, %r78, %r78;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-NOFTZ-ALLOW-NEXT:    sub.rn.f32 %r89, %r88, %r86;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p9, %r89, %r89;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p10 bra $L__BB319_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fsub_acq_rel_v8bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<11>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB319_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r9, %r8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r9, %r9;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r20, %r19, %r17;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r20, %r20;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r32, %r31, %r29;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r32, %r32;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r43, %r42, %r40;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r43, %r43;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p5 bra $L__BB319_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB319_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r55, %r54, %r52;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p6, %r55, %r55;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r66, %r65, %r63;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r66, %r66;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r78, %r77, %r75;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p8, %r78, %r78;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-FTZ-DISALLOW-NEXT:    sub.rn.ftz.f32 %r89, %r88, %r86;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r89, %r89;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p10 bra $L__BB319_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fsub_acq_rel_v8bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<11>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fsub_acq_rel_v8bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fsub_acq_rel_v8bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB319_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r9, %r8, %r6;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r9, %r9;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r20, %r19, %r17;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r20, %r20;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r32, %r31, %r29;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r32, %r32;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r43, %r42, %r40;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r43, %r43;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p5 bra $L__BB319_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-FTZ-ALLOW-NEXT:  $L__BB319_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r55, %r54, %r52;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p6, %r55, %r55;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r66, %r65, %r63;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r66, %r66;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r78, %r77, %r75;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p8, %r78, %r78;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-FTZ-ALLOW-NEXT:    sub.rn.ftz.f32 %r89, %r88, %r86;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r89, %r89;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p10 bra $L__BB319_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fsub ptr  addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
        ret <8 x bfloat> %retval
}

define <1 x bfloat> @fmin_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v1bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB320_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r11, %r8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r11, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB320_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v1bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB320_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r11, %r8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r11, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    @%p2 bra $L__BB320_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_v1bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB320_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r11, %r8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r11, %r11;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-FTZ-DISALLOW-NEXT:    @%p2 bra $L__BB320_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_v1bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fmin_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fmin_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB320_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r11, %r8, %r10;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r11, %r11;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-FTZ-ALLOW-NEXT:    @%p2 bra $L__BB320_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
        ret <1 x bfloat> %retval
}

define <2 x bfloat> @fmin_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v2bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<4>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB321_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r7, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r18, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p3 bra $L__BB321_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v2bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<4>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB321_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r7, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r18, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    @%p3 bra $L__BB321_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_v2bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<4>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB321_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-FTZ-DISALLOW-NEXT:    @%p3 bra $L__BB321_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_v2bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<4>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fmin_acq_rel_v2bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v2bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB321_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-FTZ-ALLOW-NEXT:    @%p3 bra $L__BB321_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
        ret <2 x bfloat> %retval
}

define <4 x bfloat> @fmin_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v4bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB322_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r7, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r18, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r30, %r29, %r27;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r30, %r30;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r41, %r40, %r38;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p4, %r41, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p5 bra $L__BB322_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v4bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB322_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r7, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r18, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r30, %r29, %r27;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r30, %r30;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r41, %r40, %r38;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p4, %r41, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p5 bra $L__BB322_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_v4bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB322_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r30, %r29, %r27;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r30, %r30;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r41, %r40, %r38;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r41, %r41;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p5 bra $L__BB322_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_v4bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmin_acq_rel_v4bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v4bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB322_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r30, %r29, %r27;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r30, %r30;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r41, %r40, %r38;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r41, %r41;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p5 bra $L__BB322_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
        ret <4 x bfloat> %retval
}

define <8 x bfloat> @fmin_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmin_acq_rel_v8bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<11>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB323_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r9, %r8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r9, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r20, %r19, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r20, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r32, %r31, %r29;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r32, %r32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r43, %r42, %r40;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p4, %r43, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p5 bra $L__BB323_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB323_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r55, %r54, %r52;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p6, %r55, %r55;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r66, %r65, %r63;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p7, %r66, %r66;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r78, %r77, %r75;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p8, %r78, %r78;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    min.f32 %r89, %r88, %r86;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p9, %r89, %r89;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p10 bra $L__BB323_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmin_acq_rel_v8bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<11>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB323_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r9, %r8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r9, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r20, %r19, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r20, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r32, %r31, %r29;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r32, %r32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r43, %r42, %r40;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p4, %r43, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p5 bra $L__BB323_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB323_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r55, %r54, %r52;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p6, %r55, %r55;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r66, %r65, %r63;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p7, %r66, %r66;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r78, %r77, %r75;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p8, %r78, %r78;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-NOFTZ-ALLOW-NEXT:    min.f32 %r89, %r88, %r86;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p9, %r89, %r89;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p10 bra $L__BB323_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmin_acq_rel_v8bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<11>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB323_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r9, %r8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r9, %r9;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r20, %r19, %r17;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r20, %r20;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r32, %r31, %r29;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r32, %r32;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r43, %r42, %r40;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r43, %r43;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p5 bra $L__BB323_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB323_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r55, %r54, %r52;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p6, %r55, %r55;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r66, %r65, %r63;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r66, %r66;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r78, %r77, %r75;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p8, %r78, %r78;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-FTZ-DISALLOW-NEXT:    min.ftz.f32 %r89, %r88, %r86;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r89, %r89;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p10 bra $L__BB323_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmin_acq_rel_v8bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<11>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmin_acq_rel_v8bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmin_acq_rel_v8bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB323_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r9, %r8, %r6;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r9, %r9;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r20, %r19, %r17;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r20, %r20;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r32, %r31, %r29;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r32, %r32;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r43, %r42, %r40;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r43, %r43;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p5 bra $L__BB323_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-FTZ-ALLOW-NEXT:  $L__BB323_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r55, %r54, %r52;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p6, %r55, %r55;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r66, %r65, %r63;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r66, %r66;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r78, %r77, %r75;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p8, %r78, %r78;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-FTZ-ALLOW-NEXT:    min.ftz.f32 %r89, %r88, %r86;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r89, %r89;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p10 bra $L__BB323_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmin ptr  addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
        ret <8 x bfloat> %retval
}

define <1 x bfloat> @fmax_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v1bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB324_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r11, %r8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r11, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p2 bra $L__BB324_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v1bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB324_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r11, %r8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r11, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    @%p2 bra $L__BB324_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_v1bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB324_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r11, %r8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r11, %r11;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-FTZ-DISALLOW-NEXT:    @%p2 bra $L__BB324_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_v1bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<3>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<2>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<24>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fmax_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fmax_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r23, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB324_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r23, %r1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r11, %r8, %r10;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r12, %r11, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r12, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r14, %r13, %r11;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r15, %r14, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r11, %r11;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r16, %r11, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r17, %r16, %r15, %p1;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r18, %r17, 16;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, %r1;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r20, %r23, %r2;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r21, %r20, %r19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p2, %r3, %r23;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r23, %r3;
; SM70-FTZ-ALLOW-NEXT:    @%p2 bra $L__BB324_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r22, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r22;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
        ret <1 x bfloat> %retval
}

define <2 x bfloat> @fmax_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v2bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<4>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB325_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r7, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r18, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p3 bra $L__BB325_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v2bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<4>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB325_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r7, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r18, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    @%p3 bra $L__BB325_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_v2bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<4>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB325_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-FTZ-DISALLOW-NEXT:    @%p3 bra $L__BB325_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_v2bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<4>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<5>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<27>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fmax_acq_rel_v2bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v2bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r26, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB325_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r26;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r26, %r25;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p3, %r1, %r26;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r26, %r1;
; SM70-FTZ-ALLOW-NEXT:    @%p3 bra $L__BB325_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
        ret <2 x bfloat> %retval
}

define <4 x bfloat> @fmax_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v4bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB326_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r7, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r18, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r30, %r29, %r27;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r30, %r30;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r41, %r40, %r38;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p4, %r41, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p5 bra $L__BB326_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v4bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB326_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r7, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r7, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r18, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r18, %r18;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r30, %r29, %r27;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r30, %r30;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r41, %r40, %r38;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p4, %r41, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p5 bra $L__BB326_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_v4bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB326_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r30, %r29, %r27;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r30, %r30;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r41, %r40, %r38;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r41, %r41;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p5 bra $L__BB326_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_v4bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<9>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<51>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmax_acq_rel_v4bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v4bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r26, %rs6;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r27, %r26, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB326_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r49;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r7, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r9, %r8, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r10, %r9, %r7;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r11, %r10, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r7, %r7;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r12, %r7, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r13, %r12, %r11, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r18, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r20, %r19, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r21, %r20, %r18;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r22, %r21, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r18, %r18;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r23, %r18, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r25, %r24, %r13, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r25;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r50;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs8;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r30, %r29, %r27;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r31, %r30, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r32, %r31, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r33, %r32, %r30;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r34, %r33, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r30, %r30;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r35, %r30, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r36, %r35, %r34, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r37, %rs5;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r38, %r37, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs7;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r41, %r40, %r38;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r42, %r41, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r43, %r42, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r44, %r43, %r41;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r45, %r44, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r41, %r41;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r46, %r41, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r47, %r46, %r45, %p4;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r48, %r47, %r36, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r48;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r49;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r50;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r50}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r49, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p5 bra $L__BB326_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r49, %r50};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
        ret <4 x bfloat> %retval
}

define <8 x bfloat> @fmax_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmax_acq_rel_v8bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<11>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB327_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r9, %r8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p1, %r9, %r9;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r20, %r19, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p2, %r20, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r32, %r31, %r29;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r32, %r32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r43, %r42, %r40;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p4, %r43, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p5 bra $L__BB327_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB327_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r55, %r54, %r52;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p6, %r55, %r55;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r66, %r65, %r63;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p7, %r66, %r66;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r78, %r77, %r75;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p8, %r78, %r78;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    max.f32 %r89, %r88, %r86;
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-NOFTZ-DISALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p9, %r89, %r89;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p10 bra $L__BB327_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmax_acq_rel_v8bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<11>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB327_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r9, %r8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p1, %r9, %r9;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r20, %r19, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p2, %r20, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r32, %r31, %r29;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r32, %r32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r43, %r42, %r40;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p4, %r43, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p5 bra $L__BB327_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB327_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r55, %r54, %r52;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p6, %r55, %r55;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r66, %r65, %r63;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p7, %r66, %r66;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r78, %r77, %r75;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p8, %r78, %r78;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-NOFTZ-ALLOW-NEXT:    max.f32 %r89, %r88, %r86;
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-NOFTZ-ALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p9, %r89, %r89;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p10 bra $L__BB327_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmax_acq_rel_v8bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<11>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB327_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r9, %r8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r9, %r9;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r20, %r19, %r17;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r20, %r20;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r32, %r31, %r29;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r32, %r32;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r43, %r42, %r40;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r43, %r43;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p5 bra $L__BB327_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB327_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r55, %r54, %r52;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p6, %r55, %r55;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r66, %r65, %r63;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r66, %r66;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r78, %r77, %r75;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p8, %r78, %r78;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-FTZ-DISALLOW-NEXT:    max.ftz.f32 %r89, %r88, %r86;
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-FTZ-DISALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r89, %r89;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-FTZ-DISALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-FTZ-DISALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p10 bra $L__BB327_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmax_acq_rel_v8bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<11>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<17>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<101>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmax_acq_rel_v8bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmax_acq_rel_v8bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB327_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r97;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r9, %r8, %r6;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r11, %r10, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r12, %r11, %r9;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r13, %r12, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p1, %r9, %r9;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r14, %r9, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r20, %r19, %r17;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r21, %r20, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r22, %r21, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r23, %r22, %r20;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r24, %r23, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p2, %r20, %r20;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r25, %r20, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r26, %r25, %r24, %p2;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r27, %r26, %r15, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r27;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r28, %rs6;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r29, %r28, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs7, %rs8}, %r98;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r30, %rs8;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r31, %r30, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r32, %r31, %r29;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r33, %r32, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r34, %r33, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r35, %r34, %r32;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r36, %r35, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r32, %r32;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r37, %r32, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r38, %r37, %r36, %p3;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r39, %rs5;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r40, %r39, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r41, %rs7;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r42, %r41, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r43, %r42, %r40;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r44, %r43, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r45, %r44, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r46, %r45, %r43;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r47, %r46, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p4, %r43, %r43;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r48, %r43, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r49, %r48, %r47, %p4;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r50, %r49, %r38, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r50;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r97;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r98;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p5, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r98}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r97, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p5 bra $L__BB327_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd12;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs9, %rs10}, %r3;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r4;
; SM70-FTZ-ALLOW-NEXT:  $L__BB327_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r51, %rs10;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r52, %r51, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs11, %rs12}, %r99;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r53, %rs12;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r54, %r53, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r55, %r54, %r52;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r56, %r55, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r57, %r56, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r58, %r57, %r55;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r59, %r58, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p6, %r55, %r55;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r60, %r55, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r61, %r60, %r59, %p6;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r62, %rs9;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r63, %r62, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r64, %rs11;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r65, %r64, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r66, %r65, %r63;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r67, %r66, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r68, %r67, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r69, %r68, %r66;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r70, %r69, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r66, %r66;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r71, %r66, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r72, %r71, %r70, %p7;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r73, %r72, %r61, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r73;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r74, %rs14;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r75, %r74, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r100;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r76, %rs16;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r77, %r76, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r78, %r77, %r75;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r79, %r78, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r80, %r79, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r81, %r80, %r78;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r82, %r81, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p8, %r78, %r78;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r83, %r78, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r84, %r83, %r82, %p8;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r85, %rs13;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r86, %r85, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r87, %rs15;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r88, %r87, 16;
; SM70-FTZ-ALLOW-NEXT:    max.ftz.f32 %r89, %r88, %r86;
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r90, %r89, 16;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r91, %r90, 1;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r92, %r91, %r89;
; SM70-FTZ-ALLOW-NEXT:    add.s32 %r93, %r92, 32767;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p9, %r89, %r89;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r94, %r89, 4194304;
; SM70-FTZ-ALLOW-NEXT:    selp.b32 %r95, %r94, %r93, %p9;
; SM70-FTZ-ALLOW-NEXT:    prmt.b32 %r96, %r95, %r84, 0x7632U;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r96;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r99;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r100;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p10, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r100}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r99, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p10 bra $L__BB327_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r97, %r98, %r99, %r100};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmax ptr  addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
        ret <8 x bfloat> %retval
}

define <1 x bfloat> @fminimum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_v1bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB328_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p2, %r8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, 0x7FC0, %rs4, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p4, %r8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p5 bra $L__BB328_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v1bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB328_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p2, %r8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, 0x7FC0, %rs4, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p4, %r8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    @%p5 bra $L__BB328_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v1bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB328_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, 0x7FC0, %rs4, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-FTZ-DISALLOW-NEXT:    @%p5 bra $L__BB328_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_v1bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fminimum_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fminimum_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB328_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r8, %r10;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, 0x7FC0, %rs4, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r8, %r10;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs6;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-FTZ-ALLOW-NEXT:    @%p5 bra $L__BB328_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
        ret <1 x bfloat> %retval
}

define <2 x bfloat> @fminimum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_v2bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<10>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB329_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p2, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p4, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p6, %r10, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p7, %r10, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p8, %r10, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r12, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r12, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p9 bra $L__BB329_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v2bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<10>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB329_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p2, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p4, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p6, %r10, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p7, %r10, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p8, %r10, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r12, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r12, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    @%p9 bra $L__BB329_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v2bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<10>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB329_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p6, %r10, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r10, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p8, %r10, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r12, %r11;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r12;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r12, %r1;
; SM70-FTZ-DISALLOW-NEXT:    @%p9 bra $L__BB329_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_v2bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<10>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fminimum_acq_rel_v2bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v2bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB329_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p6, %r10, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r10, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p8, %r10, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r12, %r11;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r12;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r12, %r1;
; SM70-FTZ-ALLOW-NEXT:    @%p9 bra $L__BB329_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
        ret <2 x bfloat> %retval
}

define <4 x bfloat> @fminimum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_v4bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<18>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<23>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB330_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p2, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p4, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p6, %r10, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p7, %r10, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p8, %r10, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p10, %r15, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p11, %r15, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p12, %r15, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs15;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p14, %r19, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p15, %r19, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p16, %r19, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r20, {%rs24, %rs20};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p17 bra $L__BB330_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r21, %r22};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v4bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<18>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<23>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB330_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p2, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p4, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p6, %r10, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p7, %r10, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p8, %r10, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p10, %r15, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p11, %r15, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p12, %r15, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs15;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p14, %r19, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p15, %r19, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p16, %r19, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r20, {%rs24, %rs20};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p17 bra $L__BB330_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r21, %r22};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v4bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<18>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<23>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB330_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p6, %r10, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r10, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p8, %r10, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p10, %r15, %r13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p11, %r15, %r13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p12, %r15, %r13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs15;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p14, %r19, %r17;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p15, %r19, %r17;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p16, %r19, %r17;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r20, {%rs24, %rs20};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r20;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p17 bra $L__BB330_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r21, %r22};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_v4bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<18>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<23>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fminimum_acq_rel_v4bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v4bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB330_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p6, %r10, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r10, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p8, %r10, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p10, %r15, %r13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p11, %r15, %r13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p12, %r15, %r13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs13;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs15;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p14, %r19, %r17;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p15, %r19, %r17;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p16, %r19, %r17;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r20, {%rs24, %rs20};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r20;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p17 bra $L__BB330_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r21, %r22};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
        ret <4 x bfloat> %retval
}

define <8 x bfloat> @fminimum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fminimum_acq_rel_v8bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<35>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<45>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v8bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB331_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p2, %r8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p4, %r8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r12, %r11, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p6, %r12, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p7, %r12, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p8, %r12, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p10, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p11, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p12, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r20, %rs15;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r21, %r20, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p14, %r21, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p15, %r21, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p16, %r21, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r22, {%rs24, %rs20};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p17 bra $L__BB331_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB331_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p18, %rs28, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r23, %rs26;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r24, %r23, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r25, %rs28;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r26, %r25, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p19, %r26, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p20, %r26, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs31, 0x7FC0, %rs30, %p20;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p21, %r26, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p22, %rs27, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r27, %rs25;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r28, %r27, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r29, %rs27;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r30, %r29, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p23, %r30, %r28;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p24, %r30, %r28;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs35, 0x7FC0, %rs34, %p24;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p25, %r30, %r28;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r31, {%rs36, %rs32};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r31;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r44;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p26, %rs40, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r32, %rs38;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r33, %r32, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r34, %rs40;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r35, %r34, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p27, %r35, %r33;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p28, %r35, %r33;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs43, 0x7FC0, %rs42, %p28;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p29, %r35, %r33;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p30, %rs39, -32768;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r36, %rs37;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r37, %r36, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r38, %rs39;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r39, %r38, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.lt.f32 %p31, %r39, %r37;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p32, %r39, %r37;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs47, 0x7FC0, %rs46, %p32;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p33, %r39, %r37;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r40, {%rs48, %rs44};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r40;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p34 bra $L__BB331_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fminimum_acq_rel_v8bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<35>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<45>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v8bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB331_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p2, %r8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p4, %r8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r12, %r11, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p6, %r12, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p7, %r12, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p8, %r12, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p10, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p11, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p12, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r20, %rs15;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r21, %r20, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p14, %r21, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p15, %r21, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p16, %r21, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r22, {%rs24, %rs20};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r22;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p17 bra $L__BB331_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB331_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p18, %rs28, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r23, %rs26;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r24, %r23, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r25, %rs28;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r26, %r25, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p19, %r26, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p20, %r26, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs31, 0x7FC0, %rs30, %p20;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p21, %r26, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p22, %rs27, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r27, %rs25;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r28, %r27, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r29, %rs27;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r30, %r29, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p23, %r30, %r28;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p24, %r30, %r28;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs35, 0x7FC0, %rs34, %p24;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p25, %r30, %r28;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r31, {%rs36, %rs32};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r31;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r44;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p26, %rs40, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r32, %rs38;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r33, %r32, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r34, %rs40;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r35, %r34, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p27, %r35, %r33;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p28, %r35, %r33;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs43, 0x7FC0, %rs42, %p28;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p29, %r35, %r33;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p30, %rs39, -32768;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r36, %rs37;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r37, %r36, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r38, %rs39;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r39, %r38, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.lt.f32 %p31, %r39, %r37;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p32, %r39, %r37;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs47, 0x7FC0, %rs46, %p32;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p33, %r39, %r37;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r40, {%rs48, %rs44};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r40;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p34 bra $L__BB331_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fminimum_acq_rel_v8bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<35>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<45>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v8bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB331_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r12, %r11, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p6, %r12, %r10;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r12, %r10;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p8, %r12, %r10;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p10, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p11, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p12, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r20, %rs15;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r21, %r20, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p14, %r21, %r19;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p15, %r21, %r19;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p16, %r21, %r19;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r22, {%rs24, %rs20};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r22;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p17 bra $L__BB331_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB331_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r43;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p18, %rs28, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r23, %rs26;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r24, %r23, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r25, %rs28;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r26, %r25, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p19, %r26, %r24;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p20, %r26, %r24;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs31, 0x7FC0, %rs30, %p20;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p21, %r26, %r24;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p22, %rs27, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r27, %rs25;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r28, %r27, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r29, %rs27;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r30, %r29, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p23, %r30, %r28;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p24, %r30, %r28;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs35, 0x7FC0, %rs34, %p24;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p25, %r30, %r28;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r31, {%rs36, %rs32};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r31;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r44;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p26, %rs40, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r32, %rs38;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r33, %r32, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r34, %rs40;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r35, %r34, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p27, %r35, %r33;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p28, %r35, %r33;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs43, 0x7FC0, %rs42, %p28;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p29, %r35, %r33;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p30, %rs39, -32768;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r36, %rs37;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r37, %r36, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r38, %rs39;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r39, %r38, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.lt.ftz.f32 %p31, %r39, %r37;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p32, %r39, %r37;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs47, 0x7FC0, %rs46, %p32;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p33, %r39, %r37;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r40, {%rs48, %rs44};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r40;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p34 bra $L__BB331_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fminimum_acq_rel_v8bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<35>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<45>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fminimum_acq_rel_v8bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fminimum_acq_rel_v8bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB331_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p2, %r8, %r6;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r6;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r8, %r6;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r12, %r11, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p6, %r12, %r10;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r12, %r10;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p8, %r12, %r10;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs14;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs16;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p10, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p11, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p12, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs13;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r20, %rs15;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r21, %r20, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p14, %r21, %r19;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p15, %r21, %r19;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p16, %r21, %r19;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r22, {%rs24, %rs20};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r22;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p17 bra $L__BB331_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-FTZ-ALLOW-NEXT:  $L__BB331_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r43;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p18, %rs28, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r23, %rs26;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r24, %r23, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r25, %rs28;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r26, %r25, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p19, %r26, %r24;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p20, %r26, %r24;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs31, 0x7FC0, %rs30, %p20;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p21, %r26, %r24;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p22, %rs27, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r27, %rs25;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r28, %r27, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r29, %rs27;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r30, %r29, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p23, %r30, %r28;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p24, %r30, %r28;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs35, 0x7FC0, %rs34, %p24;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p25, %r30, %r28;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r31, {%rs36, %rs32};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r31;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r44;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p26, %rs40, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r32, %rs38;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r33, %r32, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r34, %rs40;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r35, %r34, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p27, %r35, %r33;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p28, %r35, %r33;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs43, 0x7FC0, %rs42, %p28;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p29, %r35, %r33;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p30, %rs39, -32768;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r36, %rs37;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r37, %r36, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r38, %rs39;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r39, %r38, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.lt.ftz.f32 %p31, %r39, %r37;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p32, %r39, %r37;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs47, 0x7FC0, %rs46, %p32;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p33, %r39, %r37;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r40, {%rs48, %rs44};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r40;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p34 bra $L__BB331_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fminimum ptr  addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
        ret <8 x bfloat> %retval
}

define <1 x bfloat> @fmaximum_acq_rel_v1bf16_global_cta(ptr addrspace(1) %addr, <1 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_v1bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB332_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p2, %r8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, 0x7FC0, %rs4, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p4, %r8, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs6;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p5 bra $L__BB332_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v1bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB332_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p2, %r8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, 0x7FC0, %rs4, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p4, %r8, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs6;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    @%p5 bra $L__BB332_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v1bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-DISALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB332_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, 0x7FC0, %rs4, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r8, %r10;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs6;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-FTZ-DISALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-FTZ-DISALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-FTZ-DISALLOW-NEXT:    @%p5 bra $L__BB332_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v1bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<6>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<7>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<17>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<3>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd2, [fmaximum_acq_rel_v1bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.b16 %rs1, [fmaximum_acq_rel_v1bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r4, %rd2;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r5, %r4, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r1, %r5, 3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, 65535, %r1;
; SM70-FTZ-ALLOW-NEXT:    not.b32 %r2, %r6;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r16, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB332_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r7, %r16, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u16.u32 %rs2, %r7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs2, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs3, %rs2, %rs1, %p1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r8, %r10;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs4, %rs2, %rs1, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r10;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, 0x7FC0, %rs4, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r8, %r10;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs3, %rs5, %p4;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs6;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r12, %r11, %r1;
; SM70-FTZ-ALLOW-NEXT:    and.b32 %r13, %r16, %r2;
; SM70-FTZ-ALLOW-NEXT:    or.b32 %r14, %r13, %r12;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r16, %r14;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p5, %r3, %r16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r16, %r3;
; SM70-FTZ-ALLOW-NEXT:    @%p5 bra $L__BB332_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    shr.u32 %r15, %r3, %r1;
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b16 [func_retval0], %r15;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <1 x bfloat> %val syncscope("block") acq_rel
        ret <1 x bfloat> %retval
}

define <2 x bfloat> @fmaximum_acq_rel_v2bf16_global_cta(ptr addrspace(1) %addr, <2 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<10>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB333_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p2, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p4, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p6, %r10, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p7, %r10, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p8, %r10, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r12, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r12;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r12, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p9 bra $L__BB333_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<10>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB333_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p2, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p4, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p6, %r10, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p7, %r10, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p8, %r10, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r12, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r12;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r12, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    @%p9 bra $L__BB333_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<10>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB333_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p6, %r10, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r10, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p8, %r10, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r12, %r11;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r12;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r12, %r1;
; SM70-FTZ-DISALLOW-NEXT:    @%p9 bra $L__BB333_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v2bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<10>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<13>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<2>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b32 %r2, [fmaximum_acq_rel_v2bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v2bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b32 %r12, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB333_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p6, %r10, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r10, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p8, %r10, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r12, %r11;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b32 %p9, %r1, %r12;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r12, %r1;
; SM70-FTZ-ALLOW-NEXT:    @%p9 bra $L__BB333_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <2 x bfloat> %val syncscope("block") acq_rel
        ret <2 x bfloat> %retval
}

define <4 x bfloat> @fmaximum_acq_rel_v4bf16_global_cta(ptr addrspace(1) %addr, <4 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_v4bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<18>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<23>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB334_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p2, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p4, %r6, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p6, %r10, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p7, %r10, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p8, %r10, %r8;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p10, %r15, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p11, %r15, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p12, %r15, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs15;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p14, %r19, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p15, %r19, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p16, %r19, %r17;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r20, {%rs24, %rs20};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r20;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p17 bra $L__BB334_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r21, %r22};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v4bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<18>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<23>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB334_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p2, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p4, %r6, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p6, %r10, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p7, %r10, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p8, %r10, %r8;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p10, %r15, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p11, %r15, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p12, %r15, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs15;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p14, %r19, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p15, %r19, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p16, %r19, %r17;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r20, {%rs24, %rs20};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r20;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p17 bra $L__BB334_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r21, %r22};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v4bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<18>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<23>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r13, %r12, 16;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB334_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r6, %r4;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p6, %r10, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r10, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p8, %r10, %r8;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p10, %r15, %r13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p11, %r15, %r13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p12, %r15, %r13;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs15;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p14, %r19, %r17;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p15, %r19, %r17;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p16, %r19, %r17;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r20, {%rs24, %rs20};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r20;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p17 bra $L__BB334_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r21, %r22};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v4bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<18>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<25>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<23>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<12>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.v2.b32 {%r1, %r2}, [fmaximum_acq_rel_v4bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v4bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r3, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r4, %r3, 16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r12, %rs14;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r13, %r12, 16;
; SM70-FTZ-ALLOW-NEXT:  $L__BB334_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r21;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r6, %r4;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p6, %r10, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r10, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p8, %r10, %r8;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r11, {%rs12, %rs8};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r11;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r22;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs16;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p10, %r15, %r13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p11, %r15, %r13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p12, %r15, %r13;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs13;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs15;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p14, %r19, %r17;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p15, %r19, %r17;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p16, %r19, %r17;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r20, {%rs24, %rs20};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r20;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r21;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r22;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r21, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p17 bra $L__BB334_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v2.b32 [func_retval0], {%r21, %r22};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <4 x bfloat> %val syncscope("block") acq_rel
        ret <4 x bfloat> %retval
}

define <8 x bfloat> @fmaximum_acq_rel_v8bf16_global_cta(ptr addrspace(1) %addr, <8 x bfloat> %val) {
; SM70-NOFTZ-DISALLOW-LABEL: fmaximum_acq_rel_v8bf16_global_cta(
; SM70-NOFTZ-DISALLOW:       {
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .pred %p<35>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b32 %r<45>;
; SM70-NOFTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-DISALLOW-EMPTY:
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v8bf16_global_cta_param_0];
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB335_1: // %atomicrmw.start
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p2, %r8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p4, %r8, %r6;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r12, %r11, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p6, %r12, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p7, %r12, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p8, %r12, %r10;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs14;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs16;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p10, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p11, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p12, %r17, %r15;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs13;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r20, %rs15;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r21, %r20, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p14, %r21, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p15, %r21, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p16, %r21, %r19;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r22, {%rs24, %rs20};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r22;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p17 bra $L__BB335_1;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-NOFTZ-DISALLOW-NEXT:  $L__BB335_3: // %atomicrmw.start2
; SM70-NOFTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p18, %rs28, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r23, %rs26;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r24, %r23, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r25, %rs28;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r26, %r25, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p19, %r26, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p20, %r26, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs31, 0x7FC0, %rs30, %p20;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p21, %r26, %r24;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p22, %rs27, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r27, %rs25;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r28, %r27, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r29, %rs27;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r30, %r29, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p23, %r30, %r28;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p24, %r30, %r28;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs35, 0x7FC0, %rs34, %p24;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p25, %r30, %r28;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r31, {%rs36, %rs32};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r31;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r44;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p26, %rs40, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r32, %rs38;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r33, %r32, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r34, %rs40;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r35, %r34, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p27, %r35, %r33;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p28, %r35, %r33;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs43, 0x7FC0, %rs42, %p28;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p29, %r35, %r33;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.b16 %p30, %rs39, 0;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r36, %rs37;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r37, %r36, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u16 %r38, %rs39;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b32 %r39, %r38, 16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.gt.f32 %p31, %r39, %r37;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.nan.f32 %p32, %r39, %r37;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs47, 0x7FC0, %rs46, %p32;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.eq.f32 %p33, %r39, %r37;
; SM70-NOFTZ-DISALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-NOFTZ-DISALLOW-NEXT:    mov.b32 %r40, {%rs48, %rs44};
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r40;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-NOFTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-DISALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-NOFTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-NOFTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-NOFTZ-DISALLOW-NEXT:    @%p34 bra $L__BB335_3;
; SM70-NOFTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-NOFTZ-DISALLOW-NEXT:    ret;
;
; SM70-NOFTZ-ALLOW-LABEL: fmaximum_acq_rel_v8bf16_global_cta(
; SM70-NOFTZ-ALLOW:       {
; SM70-NOFTZ-ALLOW-NEXT:    .reg .pred %p<35>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b32 %r<45>;
; SM70-NOFTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-NOFTZ-ALLOW-EMPTY:
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.0:
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v8bf16_global_cta_param_0];
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB335_1: // %atomicrmw.start
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p2, %r8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p3, %r8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p4, %r8, %r6;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r12, %r11, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p6, %r12, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p7, %r12, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p8, %r12, %r10;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs14;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs16;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p10, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p11, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p12, %r17, %r15;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs13;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r20, %rs15;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r21, %r20, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p14, %r21, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p15, %r21, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p16, %r21, %r19;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r22, {%rs24, %rs20};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r22;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-NOFTZ-ALLOW-NEXT:    @%p17 bra $L__BB335_1;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NOFTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-NOFTZ-ALLOW-NEXT:  $L__BB335_3: // %atomicrmw.start2
; SM70-NOFTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p18, %rs28, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r23, %rs26;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r24, %r23, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r25, %rs28;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r26, %r25, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p19, %r26, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p20, %r26, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs31, 0x7FC0, %rs30, %p20;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p21, %r26, %r24;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p22, %rs27, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r27, %rs25;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r28, %r27, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r29, %rs27;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r30, %r29, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p23, %r30, %r28;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p24, %r30, %r28;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs35, 0x7FC0, %rs34, %p24;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p25, %r30, %r28;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r31, {%rs36, %rs32};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r31;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r44;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p26, %rs40, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r32, %rs38;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r33, %r32, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r34, %rs40;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r35, %r34, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p27, %r35, %r33;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p28, %r35, %r33;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs43, 0x7FC0, %rs42, %p28;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p29, %r35, %r33;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.b16 %p30, %rs39, 0;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r36, %rs37;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r37, %r36, 16;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u16 %r38, %rs39;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b32 %r39, %r38, 16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.gt.f32 %p31, %r39, %r37;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-NOFTZ-ALLOW-NEXT:    setp.nan.f32 %p32, %r39, %r37;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs47, 0x7FC0, %rs46, %p32;
; SM70-NOFTZ-ALLOW-NEXT:    setp.eq.f32 %p33, %r39, %r37;
; SM70-NOFTZ-ALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-NOFTZ-ALLOW-NEXT:    mov.b32 %r40, {%rs48, %rs44};
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r40;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-NOFTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-NOFTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-NOFTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-NOFTZ-ALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-NOFTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-NOFTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-NOFTZ-ALLOW-NEXT:    @%p34 bra $L__BB335_3;
; SM70-NOFTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NOFTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-NOFTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-NOFTZ-ALLOW-NEXT:    ret;
;
; SM70-FTZ-DISALLOW-LABEL: fmaximum_acq_rel_v8bf16_global_cta(
; SM70-FTZ-DISALLOW:       {
; SM70-FTZ-DISALLOW-NEXT:    .reg .pred %p<35>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b32 %r<45>;
; SM70-FTZ-DISALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-DISALLOW-EMPTY:
; SM70-FTZ-DISALLOW-NEXT:  // %bb.0:
; SM70-FTZ-DISALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v8bf16_global_cta_param_0];
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB335_1: // %atomicrmw.start
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r8, %r6;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r12, %r11, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p6, %r12, %r10;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r12, %r10;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p8, %r12, %r10;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r14, %rs14;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r16, %rs16;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p10, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p11, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p12, %r17, %r15;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r18, %rs13;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r20, %rs15;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r21, %r20, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p14, %r21, %r19;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p15, %r21, %r19;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p16, %r21, %r19;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r22, {%rs24, %rs20};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd4, %r22;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-FTZ-DISALLOW-NEXT:    @%p17 bra $L__BB335_1;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-DISALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-FTZ-DISALLOW-NEXT:  $L__BB335_3: // %atomicrmw.start2
; SM70-FTZ-DISALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r43;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p18, %rs28, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r23, %rs26;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r24, %r23, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r25, %rs28;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r26, %r25, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p19, %r26, %r24;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p20, %r26, %r24;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs31, 0x7FC0, %rs30, %p20;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p21, %r26, %r24;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p22, %rs27, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r27, %rs25;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r28, %r27, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r29, %rs27;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r30, %r29, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p23, %r30, %r28;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p24, %r30, %r28;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs35, 0x7FC0, %rs34, %p24;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p25, %r30, %r28;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r31, {%rs36, %rs32};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd13, %r31;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r44;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p26, %rs40, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r32, %rs38;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r33, %r32, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r34, %rs40;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r35, %r34, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p27, %r35, %r33;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p28, %r35, %r33;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs43, 0x7FC0, %rs42, %p28;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p29, %r35, %r33;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.b16 %p30, %rs39, 0;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r36, %rs37;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r37, %r36, 16;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u16 %r38, %rs39;
; SM70-FTZ-DISALLOW-NEXT:    shl.b32 %r39, %r38, 16;
; SM70-FTZ-DISALLOW-NEXT:    setp.gt.ftz.f32 %p31, %r39, %r37;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-FTZ-DISALLOW-NEXT:    setp.nan.ftz.f32 %p32, %r39, %r37;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs47, 0x7FC0, %rs46, %p32;
; SM70-FTZ-DISALLOW-NEXT:    setp.eq.ftz.f32 %p33, %r39, %r37;
; SM70-FTZ-DISALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-FTZ-DISALLOW-NEXT:    mov.b32 %r40, {%rs48, %rs44};
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd14, %r40;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-FTZ-DISALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-FTZ-DISALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-DISALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-DISALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-DISALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-FTZ-DISALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-FTZ-DISALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-FTZ-DISALLOW-NEXT:    @%p34 bra $L__BB335_3;
; SM70-FTZ-DISALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-DISALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-DISALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-FTZ-DISALLOW-NEXT:    ret;
;
; SM70-FTZ-ALLOW-LABEL: fmaximum_acq_rel_v8bf16_global_cta(
; SM70-FTZ-ALLOW:       {
; SM70-FTZ-ALLOW-NEXT:    .reg .pred %p<35>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b16 %rs<49>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b32 %r<45>;
; SM70-FTZ-ALLOW-NEXT:    .reg .b64 %rd<22>;
; SM70-FTZ-ALLOW-EMPTY:
; SM70-FTZ-ALLOW-NEXT:  // %bb.0:
; SM70-FTZ-ALLOW-NEXT:    ld.param.b64 %rd1, [fmaximum_acq_rel_v8bf16_global_cta_param_0];
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [fmaximum_acq_rel_v8bf16_global_cta_param_1];
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd2; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd2;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs13, %rs14}, %r2;
; SM70-FTZ-ALLOW-NEXT:  $L__BB335_1: // %atomicrmw.start
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs3, %rs4}, %r41;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p1, %rs4, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r5, %rs2;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r6, %r5, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r7, %rs4;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r8, %r7, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p2, %r8, %r6;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs6, %rs4, %rs2, %p2;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p3, %r8, %r6;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs7, 0x7FC0, %rs6, %p3;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p4, %r8, %r6;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs8, %rs5, %rs7, %p4;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p5, %rs3, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs9, %rs3, %rs1, %p5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r9, %rs1;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r10, %r9, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r11, %rs3;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r12, %r11, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p6, %r12, %r10;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p7, %r12, %r10;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p8, %r12, %r10;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs12, %rs9, %rs11, %p8;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r13, {%rs12, %rs8};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd3, %r13;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs15, %rs16}, %r42;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p9, %rs16, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs17, %rs16, %rs14, %p9;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r14, %rs14;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r15, %r14, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r16, %rs16;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r17, %r16, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p10, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs18, %rs16, %rs14, %p10;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p11, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs19, 0x7FC0, %rs18, %p11;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p12, %r17, %r15;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs20, %rs17, %rs19, %p12;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p13, %rs15, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs21, %rs15, %rs13, %p13;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r18, %rs13;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r19, %r18, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r20, %rs15;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r21, %r20, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p14, %r21, %r19;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs22, %rs15, %rs13, %p14;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p15, %r21, %r19;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs23, 0x7FC0, %rs22, %p15;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p16, %r21, %r19;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs24, %rs21, %rs23, %p16;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r22, {%rs24, %rs20};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd4, %r22;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd7, %r41;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd8, %r42;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p17, %rd11, %rd10;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r42}, %rd11; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r41, %rd11;
; SM70-FTZ-ALLOW-NEXT:    @%p17 bra $L__BB335_1;
; SM70-FTZ-ALLOW-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-FTZ-ALLOW-NEXT:    ld.relaxed.cta.global.b64 %rd12, [%rd1+8];
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd12; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd12;
; SM70-FTZ-ALLOW-NEXT:  $L__BB335_3: // %atomicrmw.start2
; SM70-FTZ-ALLOW-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs25, %rs26}, %r3;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs27, %rs28}, %r43;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p18, %rs28, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs29, %rs28, %rs26, %p18;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r23, %rs26;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r24, %r23, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r25, %rs28;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r26, %r25, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p19, %r26, %r24;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs30, %rs28, %rs26, %p19;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p20, %r26, %r24;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs31, 0x7FC0, %rs30, %p20;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p21, %r26, %r24;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs32, %rs29, %rs31, %p21;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p22, %rs27, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs33, %rs27, %rs25, %p22;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r27, %rs25;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r28, %r27, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r29, %rs27;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r30, %r29, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p23, %r30, %r28;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs34, %rs27, %rs25, %p23;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p24, %r30, %r28;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs35, 0x7FC0, %rs34, %p24;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p25, %r30, %r28;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs36, %rs33, %rs35, %p25;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r31, {%rs36, %rs32};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd13, %r31;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs37, %rs38}, %r4;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 {%rs39, %rs40}, %r44;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p26, %rs40, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs41, %rs40, %rs38, %p26;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r32, %rs38;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r33, %r32, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r34, %rs40;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r35, %r34, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p27, %r35, %r33;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs42, %rs40, %rs38, %p27;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p28, %r35, %r33;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs43, 0x7FC0, %rs42, %p28;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p29, %r35, %r33;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs44, %rs41, %rs43, %p29;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.b16 %p30, %rs39, 0;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs45, %rs39, %rs37, %p30;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r36, %rs37;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r37, %r36, 16;
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u16 %r38, %rs39;
; SM70-FTZ-ALLOW-NEXT:    shl.b32 %r39, %r38, 16;
; SM70-FTZ-ALLOW-NEXT:    setp.gt.ftz.f32 %p31, %r39, %r37;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs46, %rs39, %rs37, %p31;
; SM70-FTZ-ALLOW-NEXT:    setp.nan.ftz.f32 %p32, %r39, %r37;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs47, 0x7FC0, %rs46, %p32;
; SM70-FTZ-ALLOW-NEXT:    setp.eq.ftz.f32 %p33, %r39, %r37;
; SM70-FTZ-ALLOW-NEXT:    selp.b16 %rs48, %rs45, %rs47, %p33;
; SM70-FTZ-ALLOW-NEXT:    mov.b32 %r40, {%rs48, %rs44};
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd14, %r40;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd15, %rd14, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd16, %rd13, %rd15;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd17, %r43;
; SM70-FTZ-ALLOW-NEXT:    cvt.u64.u32 %rd18, %r44;
; SM70-FTZ-ALLOW-NEXT:    shl.b64 %rd19, %rd18, 32;
; SM70-FTZ-ALLOW-NEXT:    or.b64 %rd20, %rd17, %rd19;
; SM70-FTZ-ALLOW-NEXT:    atom.relaxed.cta.global.cas.b64 %rd21, [%rd1+8], %rd20, %rd16;
; SM70-FTZ-ALLOW-NEXT:    setp.ne.b64 %p34, %rd21, %rd20;
; SM70-FTZ-ALLOW-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r44}, %rd21; }
; SM70-FTZ-ALLOW-NEXT:    cvt.u32.u64 %r43, %rd21;
; SM70-FTZ-ALLOW-NEXT:    @%p34 bra $L__BB335_3;
; SM70-FTZ-ALLOW-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-FTZ-ALLOW-NEXT:    fence.acq_rel.cta;
; SM70-FTZ-ALLOW-NEXT:    st.param.v4.b32 [func_retval0], {%r41, %r42, %r43, %r44};
; SM70-FTZ-ALLOW-NEXT:    ret;
        %retval = atomicrmw elementwise fmaximum ptr  addrspace(1) %addr, <8 x bfloat> %val syncscope("block") acq_rel
        ret <8 x bfloat> %retval
}

define <1 x i8> @add_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: add_monotonic_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [add_monotonic_v1i8_global_cta_param_0];
; SM70-NEXT:    ld.param.b8 %r6, [add_monotonic_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:    and.b32 %r8, %r7, 3;
; SM70-NEXT:    shl.b32 %r1, %r8, 3;
; SM70-NEXT:    shl.b32 %r2, 255, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    shl.b32 %r4, %r6, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:  $L__BB336_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    add.s32 %r9, %r14, %r4;
; SM70-NEXT:    and.b32 %r10, %r9, %r2;
; SM70-NEXT:    and.b32 %r11, %r14, %r3;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r14;
; SM70-NEXT:    mov.b32 %r14, %r5;
; SM70-NEXT:    @%p1 bra $L__BB336_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r5, %r1;
; SM70-NEXT:    st.param.b8 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
        ret <1 x i8> %retval
}

define <2 x i8> @add_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: add_monotonic_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<12>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [add_monotonic_v2i8_global_cta_param_1];
; SM70-NEXT:    mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT:    ld.param.b64 %rd2, [add_monotonic_v2i8_global_cta_param_0];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    and.b32 %r6, %r5, 3;
; SM70-NEXT:    shl.b32 %r1, %r6, 3;
; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r7;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
; SM70-NEXT:  $L__BB337_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r8, %r14, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r8;
; SM70-NEXT:    shr.u16 %rs4, %rs3, 8;
; SM70-NEXT:    add.s16 %rs7, %rs4, %rs6;
; SM70-NEXT:    add.s16 %rs8, %rs3, %rs5;
; SM70-NEXT:    and.b16 %rs9, %rs8, 255;
; SM70-NEXT:    shl.b16 %rs10, %rs7, 8;
; SM70-NEXT:    or.b16 %rs11, %rs9, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r9, %rs11;
; SM70-NEXT:    shl.b32 %r10, %r9, %r1;
; SM70-NEXT:    and.b32 %r11, %r14, %r2;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r14;
; SM70-NEXT:    mov.b32 %r14, %r3;
; SM70-NEXT:    @%p1 bra $L__BB337_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r3, %r1;
; SM70-NEXT:    st.param.b16 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
        ret <2 x i8> %retval
}

define <4 x i8> @add_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: add_monotonic_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<19>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [add_monotonic_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [add_monotonic_v4i8_global_cta_param_0];
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
; SM70-NEXT:  $L__BB338_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
; SM70-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
; SM70-NEXT:    add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
; SM70-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
; SM70-NEXT:    add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
; SM70-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
; SM70-NEXT:    add.s16 %rs12, %rs11, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r15, %rs12;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r18;
; SM70-NEXT:    mov.b32 %r18, %r1;
; SM70-NEXT:    @%p1 bra $L__BB338_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
        ret <4 x i8> %retval
}

define <8 x i8> @add_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: add_monotonic_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<35>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [add_monotonic_v8i8_global_cta_param_0];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NEXT:  $L__BB339_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r3, %r1, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
; SM70-NEXT:    prmt.b32 %r6, %r1, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
; SM70-NEXT:    prmt.b32 %r7, %r33, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
; SM70-NEXT:    add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
; SM70-NEXT:    prmt.b32 %r10, %r1, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
; SM70-NEXT:    prmt.b32 %r11, %r33, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
; SM70-NEXT:    add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
; SM70-NEXT:    prmt.b32 %r13, %r1, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
; SM70-NEXT:    prmt.b32 %r14, %r33, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
; SM70-NEXT:    add.s16 %rs12, %rs11, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r15, %rs12;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r17;
; SM70-NEXT:    prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r18;
; SM70-NEXT:    prmt.b32 %r19, %r34, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs14, %r19;
; SM70-NEXT:    add.s16 %rs15, %rs14, %rs13;
; SM70-NEXT:    cvt.u32.u16 %r20, %rs15;
; SM70-NEXT:    prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs16, %r21;
; SM70-NEXT:    prmt.b32 %r22, %r34, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs17, %r22;
; SM70-NEXT:    add.s16 %rs18, %rs17, %rs16;
; SM70-NEXT:    cvt.u32.u16 %r23, %rs18;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r20, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs19, %r25;
; SM70-NEXT:    prmt.b32 %r26, %r34, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs20, %r26;
; SM70-NEXT:    add.s16 %rs21, %rs20, %rs19;
; SM70-NEXT:    cvt.u32.u16 %r27, %rs21;
; SM70-NEXT:    prmt.b32 %r28, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs22, %r28;
; SM70-NEXT:    prmt.b32 %r29, %r34, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs23, %r29;
; SM70-NEXT:    add.s16 %rs24, %rs23, %rs22;
; SM70-NEXT:    cvt.u32.u16 %r30, %rs24;
; SM70-NEXT:    prmt.b32 %r31, %r30, %r27, 0x3340U;
; SM70-NEXT:    prmt.b32 %r32, %r31, %r24, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r32;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB339_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
        ret <8 x i8> %retval
}

define <1 x i8> @add_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: add_acquire_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [add_acquire_v1i8_global_cta_param_0];
; SM70-NEXT:    ld.param.b8 %r6, [add_acquire_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:    and.b32 %r8, %r7, 3;
; SM70-NEXT:    shl.b32 %r1, %r8, 3;
; SM70-NEXT:    shl.b32 %r2, 255, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    shl.b32 %r4, %r6, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:  $L__BB340_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    add.s32 %r9, %r14, %r4;
; SM70-NEXT:    and.b32 %r10, %r9, %r2;
; SM70-NEXT:    and.b32 %r11, %r14, %r3;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r14;
; SM70-NEXT:    mov.b32 %r14, %r5;
; SM70-NEXT:    @%p1 bra $L__BB340_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r5, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
        ret <1 x i8> %retval
}

define <2 x i8> @add_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: add_acquire_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<12>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [add_acquire_v2i8_global_cta_param_1];
; SM70-NEXT:    mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT:    ld.param.b64 %rd2, [add_acquire_v2i8_global_cta_param_0];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    and.b32 %r6, %r5, 3;
; SM70-NEXT:    shl.b32 %r1, %r6, 3;
; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r7;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
; SM70-NEXT:  $L__BB341_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r8, %r14, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r8;
; SM70-NEXT:    shr.u16 %rs4, %rs3, 8;
; SM70-NEXT:    add.s16 %rs7, %rs4, %rs6;
; SM70-NEXT:    add.s16 %rs8, %rs3, %rs5;
; SM70-NEXT:    and.b16 %rs9, %rs8, 255;
; SM70-NEXT:    shl.b16 %rs10, %rs7, 8;
; SM70-NEXT:    or.b16 %rs11, %rs9, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r9, %rs11;
; SM70-NEXT:    shl.b32 %r10, %r9, %r1;
; SM70-NEXT:    and.b32 %r11, %r14, %r2;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r14;
; SM70-NEXT:    mov.b32 %r14, %r3;
; SM70-NEXT:    @%p1 bra $L__BB341_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r3, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
        ret <2 x i8> %retval
}

define <4 x i8> @add_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: add_acquire_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<19>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [add_acquire_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [add_acquire_v4i8_global_cta_param_0];
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
; SM70-NEXT:  $L__BB342_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
; SM70-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
; SM70-NEXT:    add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
; SM70-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
; SM70-NEXT:    add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
; SM70-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
; SM70-NEXT:    add.s16 %rs12, %rs11, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r15, %rs12;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r18;
; SM70-NEXT:    mov.b32 %r18, %r1;
; SM70-NEXT:    @%p1 bra $L__BB342_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
        ret <4 x i8> %retval
}

define <8 x i8> @add_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: add_acquire_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<35>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [add_acquire_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [add_acquire_v8i8_global_cta_param_0];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NEXT:  $L__BB343_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r3, %r1, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
; SM70-NEXT:    prmt.b32 %r6, %r1, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
; SM70-NEXT:    prmt.b32 %r7, %r33, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
; SM70-NEXT:    add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
; SM70-NEXT:    prmt.b32 %r10, %r1, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
; SM70-NEXT:    prmt.b32 %r11, %r33, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
; SM70-NEXT:    add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
; SM70-NEXT:    prmt.b32 %r13, %r1, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
; SM70-NEXT:    prmt.b32 %r14, %r33, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
; SM70-NEXT:    add.s16 %rs12, %rs11, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r15, %rs12;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r17;
; SM70-NEXT:    prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r18;
; SM70-NEXT:    prmt.b32 %r19, %r34, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs14, %r19;
; SM70-NEXT:    add.s16 %rs15, %rs14, %rs13;
; SM70-NEXT:    cvt.u32.u16 %r20, %rs15;
; SM70-NEXT:    prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs16, %r21;
; SM70-NEXT:    prmt.b32 %r22, %r34, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs17, %r22;
; SM70-NEXT:    add.s16 %rs18, %rs17, %rs16;
; SM70-NEXT:    cvt.u32.u16 %r23, %rs18;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r20, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs19, %r25;
; SM70-NEXT:    prmt.b32 %r26, %r34, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs20, %r26;
; SM70-NEXT:    add.s16 %rs21, %rs20, %rs19;
; SM70-NEXT:    cvt.u32.u16 %r27, %rs21;
; SM70-NEXT:    prmt.b32 %r28, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs22, %r28;
; SM70-NEXT:    prmt.b32 %r29, %r34, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs23, %r29;
; SM70-NEXT:    add.s16 %rs24, %rs23, %rs22;
; SM70-NEXT:    cvt.u32.u16 %r30, %rs24;
; SM70-NEXT:    prmt.b32 %r31, %r30, %r27, 0x3340U;
; SM70-NEXT:    prmt.b32 %r32, %r31, %r24, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r32;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB343_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
        ret <8 x i8> %retval
}

define <1 x i8> @add_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: add_release_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [add_release_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b8 %r6, [add_release_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:    and.b32 %r8, %r7, 3;
; SM70-NEXT:    shl.b32 %r1, %r8, 3;
; SM70-NEXT:    shl.b32 %r2, 255, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    shl.b32 %r4, %r6, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:  $L__BB344_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    add.s32 %r9, %r14, %r4;
; SM70-NEXT:    and.b32 %r10, %r9, %r2;
; SM70-NEXT:    and.b32 %r11, %r14, %r3;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r14;
; SM70-NEXT:    mov.b32 %r14, %r5;
; SM70-NEXT:    @%p1 bra $L__BB344_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r5, %r1;
; SM70-NEXT:    st.param.b8 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") release
        ret <1 x i8> %retval
}

define <2 x i8> @add_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: add_release_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<12>;
; SM70-NEXT:    .reg .b32 %r<15>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [add_release_v2i8_global_cta_param_1];
; SM70-NEXT:    mov.b32 %r4, {%rs1, %rs2};
; SM70-NEXT:    ld.param.b64 %rd2, [add_release_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:    and.b32 %r6, %r5, 3;
; SM70-NEXT:    shl.b32 %r1, %r6, 3;
; SM70-NEXT:    shl.b32 %r7, 65535, %r1;
; SM70-NEXT:    not.b32 %r2, %r7;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r14, [%rd1];
; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
; SM70-NEXT:  $L__BB345_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    shr.u32 %r8, %r14, %r1;
; SM70-NEXT:    cvt.u16.u32 %rs3, %r8;
; SM70-NEXT:    shr.u16 %rs4, %rs3, 8;
; SM70-NEXT:    add.s16 %rs7, %rs4, %rs6;
; SM70-NEXT:    add.s16 %rs8, %rs3, %rs5;
; SM70-NEXT:    and.b16 %rs9, %rs8, 255;
; SM70-NEXT:    shl.b16 %rs10, %rs7, 8;
; SM70-NEXT:    or.b16 %rs11, %rs9, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r9, %rs11;
; SM70-NEXT:    shl.b32 %r10, %r9, %r1;
; SM70-NEXT:    and.b32 %r11, %r14, %r2;
; SM70-NEXT:    or.b32 %r12, %r11, %r10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12;
; SM70-NEXT:    setp.ne.b32 %p1, %r3, %r14;
; SM70-NEXT:    mov.b32 %r14, %r3;
; SM70-NEXT:    @%p1 bra $L__BB345_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r13, %r3, %r1;
; SM70-NEXT:    st.param.b16 [func_retval0], %r13;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") release
        ret <2 x i8> %retval
}

define <4 x i8> @add_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: add_release_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<13>;
; SM70-NEXT:    .reg .b32 %r<19>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [add_release_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [add_release_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r18, [%rd1];
; SM70-NEXT:    prmt.b32 %r3, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    prmt.b32 %r6, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
; SM70-NEXT:    prmt.b32 %r10, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
; SM70-NEXT:    prmt.b32 %r13, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
; SM70-NEXT:  $L__BB346_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r4, %r18, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
; SM70-NEXT:    prmt.b32 %r7, %r18, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
; SM70-NEXT:    add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
; SM70-NEXT:    prmt.b32 %r11, %r18, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
; SM70-NEXT:    add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
; SM70-NEXT:    prmt.b32 %r14, %r18, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
; SM70-NEXT:    add.s16 %rs12, %rs11, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r15, %rs12;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r18, %r17;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r18;
; SM70-NEXT:    mov.b32 %r18, %r1;
; SM70-NEXT:    @%p1 bra $L__BB346_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") release
        ret <4 x i8> %retval
}

define <8 x i8> @add_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: add_release_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b16 %rs<25>;
; SM70-NEXT:    .reg .b32 %r<35>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [add_release_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [add_release_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd2;
; SM70-NEXT:  $L__BB347_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    prmt.b32 %r3, %r1, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs1, %r3;
; SM70-NEXT:    prmt.b32 %r4, %r33, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs2, %r4;
; SM70-NEXT:    add.s16 %rs3, %rs2, %rs1;
; SM70-NEXT:    cvt.u32.u16 %r5, %rs3;
; SM70-NEXT:    prmt.b32 %r6, %r1, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs4, %r6;
; SM70-NEXT:    prmt.b32 %r7, %r33, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs5, %r7;
; SM70-NEXT:    add.s16 %rs6, %rs5, %rs4;
; SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
; SM70-NEXT:    prmt.b32 %r9, %r8, %r5, 0x3340U;
; SM70-NEXT:    prmt.b32 %r10, %r1, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs7, %r10;
; SM70-NEXT:    prmt.b32 %r11, %r33, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs8, %r11;
; SM70-NEXT:    add.s16 %rs9, %rs8, %rs7;
; SM70-NEXT:    cvt.u32.u16 %r12, %rs9;
; SM70-NEXT:    prmt.b32 %r13, %r1, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs10, %r13;
; SM70-NEXT:    prmt.b32 %r14, %r33, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs11, %r14;
; SM70-NEXT:    add.s16 %rs12, %rs11, %rs10;
; SM70-NEXT:    cvt.u32.u16 %r15, %rs12;
; SM70-NEXT:    prmt.b32 %r16, %r15, %r12, 0x3340U;
; SM70-NEXT:    prmt.b32 %r17, %r16, %r9, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd3, %r17;
; SM70-NEXT:    prmt.b32 %r18, %r2, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs13, %r18;
; SM70-NEXT:    prmt.b32 %r19, %r34, 0, 0x7773U;
; SM70-NEXT:    cvt.u16.u32 %rs14, %r19;
; SM70-NEXT:    add.s16 %rs15, %rs14, %rs13;
; SM70-NEXT:    cvt.u32.u16 %r20, %rs15;
; SM70-NEXT:    prmt.b32 %r21, %r2, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs16, %r21;
; SM70-NEXT:    prmt.b32 %r22, %r34, 0, 0x7772U;
; SM70-NEXT:    cvt.u16.u32 %rs17, %r22;
; SM70-NEXT:    add.s16 %rs18, %rs17, %rs16;
; SM70-NEXT:    cvt.u32.u16 %r23, %rs18;
; SM70-NEXT:    prmt.b32 %r24, %r23, %r20, 0x3340U;
; SM70-NEXT:    prmt.b32 %r25, %r2, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs19, %r25;
; SM70-NEXT:    prmt.b32 %r26, %r34, 0, 0x7771U;
; SM70-NEXT:    cvt.u16.u32 %rs20, %r26;
; SM70-NEXT:    add.s16 %rs21, %rs20, %rs19;
; SM70-NEXT:    cvt.u32.u16 %r27, %rs21;
; SM70-NEXT:    prmt.b32 %r28, %r2, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs22, %r28;
; SM70-NEXT:    prmt.b32 %r29, %r34, 0, 0x7770U;
; SM70-NEXT:    cvt.u16.u32 %rs23, %r29;
; SM70-NEXT:    add.s16 %rs24, %rs23, %rs22;
; SM70-NEXT:    cvt.u32.u16 %r30, %rs24;
; SM70-NEXT:    prmt.b32 %r31, %r30, %r27, 0x3340U;
; SM70-NEXT:    prmt.b32 %r32, %r31, %r24, 0x5410U;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r32;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r33;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r34;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd10, %rd6;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd10;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r34}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r33, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB347_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r33, %r34};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") release
        ret <8 x i8> %retval
}

define <1 x i32> @add_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: add_monotonic_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_monotonic_v1i32_global_cta_param_0];
; SM70-NEXT:    ld.param.b32 %r1, [add_monotonic_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
        ret <1 x i32> %retval
}

define <2 x i32> @add_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: add_monotonic_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_monotonic_v2i32_global_cta_param_0];
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [add_monotonic_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
        ret <2 x i32> %retval
}

define <4 x i32> @add_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: add_monotonic_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_monotonic_v4i32_global_cta_param_0];
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v4i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
        ret <4 x i32> %retval
}

define <8 x i32> @add_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: add_monotonic_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_monotonic_v8i32_global_cta_param_0];
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_monotonic_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_monotonic_v8i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
        ret <8 x i32> %retval
}

define <1 x i32> @add_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: add_acquire_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_acquire_v1i32_global_cta_param_0];
; SM70-NEXT:    ld.param.b32 %r1, [add_acquire_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
        ret <1 x i32> %retval
}

define <2 x i32> @add_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: add_acquire_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_acquire_v2i32_global_cta_param_0];
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [add_acquire_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
        ret <2 x i32> %retval
}

define <4 x i32> @add_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: add_acquire_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_acquire_v4i32_global_cta_param_0];
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v4i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
        ret <4 x i32> %retval
}

define <8 x i32> @add_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: add_acquire_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_acquire_v8i32_global_cta_param_0];
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_acquire_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_acquire_v8i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
        ret <8 x i32> %retval
}

define <1 x i32> @add_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: add_release_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<3>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_release_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [add_release_v1i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") release
        ret <1 x i32> %retval
}

define <2 x i32> @add_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: add_release_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<5>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_release_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [add_release_v2i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r3, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r4, [%rd1+4], %r2;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r4};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") release
        ret <2 x i32> %retval
}

define <4 x i32> @add_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: add_release_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_release_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v4i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r5, [%rd1], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r6, [%rd1+4], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r7, [%rd1+8], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r8, [%rd1+12], %r4;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r5, %r6, %r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") release
        ret <4 x i32> %retval
}

define <8 x i32> @add_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: add_release_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .b32 %r<17>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [add_release_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [add_release_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [add_release_v8i32_global_cta_param_1];
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r9, [%rd1], %r5;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r10, [%rd1+4], %r6;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r11, [%rd1+8], %r7;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r12, [%rd1+12], %r8;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r13, [%rd1+16], %r1;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r14, [%rd1+20], %r2;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r15, [%rd1+24], %r3;
; SM70-NEXT:    atom.relaxed.cta.global.add.u32 %r16, [%rd1+28], %r4;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r13, %r14, %r15, %r16};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise add ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") release
        ret <8 x i32> %retval
}

define <1 x i8> @nand_monotonic_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: nand_monotonic_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<16>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [nand_monotonic_v1i8_global_cta_param_0];
; SM70-NEXT:    ld.param.b8 %r6, [nand_monotonic_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:    and.b32 %r8, %r7, 3;
; SM70-NEXT:    shl.b32 %r1, %r8, 3;
; SM70-NEXT:    shl.b32 %r2, 255, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    shl.b32 %r4, %r6, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT:  $L__BB360_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r9, %r15, %r4;
; SM70-NEXT:    not.b32 %r10, %r9;
; SM70-NEXT:    and.b32 %r11, %r10, %r2;
; SM70-NEXT:    and.b32 %r12, %r15, %r3;
; SM70-NEXT:    or.b32 %r13, %r12, %r11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r15;
; SM70-NEXT:    mov.b32 %r15, %r5;
; SM70-NEXT:    @%p1 bra $L__BB360_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r14, %r5, %r1;
; SM70-NEXT:    st.param.b8 [func_retval0], %r14;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") monotonic
        ret <1 x i8> %retval
}

define <2 x i8> @nand_monotonic_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: nand_monotonic_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<16>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [nand_monotonic_v2i8_global_cta_param_0];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r6, %rd2;
; SM70-NEXT:    and.b32 %r7, %r6, 3;
; SM70-NEXT:    shl.b32 %r1, %r7, 3;
; SM70-NEXT:    shl.b32 %r2, 65535, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    ld.param.b16 %r8, [nand_monotonic_v2i8_global_cta_param_1];
; SM70-NEXT:    shl.b32 %r4, %r8, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT:  $L__BB361_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r9, %r15, %r4;
; SM70-NEXT:    not.b32 %r10, %r9;
; SM70-NEXT:    and.b32 %r11, %r10, %r2;
; SM70-NEXT:    and.b32 %r12, %r15, %r3;
; SM70-NEXT:    or.b32 %r13, %r12, %r11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r15;
; SM70-NEXT:    mov.b32 %r15, %r5;
; SM70-NEXT:    @%p1 bra $L__BB361_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r14, %r5, %r1;
; SM70-NEXT:    st.param.b16 [func_retval0], %r14;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") monotonic
        ret <2 x i8> %retval
}

define <4 x i8> @nand_monotonic_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: nand_monotonic_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<6>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [nand_monotonic_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_monotonic_v4i8_global_cta_param_0];
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT:  $L__BB362_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r3, %r5, %r2;
; SM70-NEXT:    xor.b32 %r4, %r3, -1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r5;
; SM70-NEXT:    mov.b32 %r5, %r1;
; SM70-NEXT:    @%p1 bra $L__BB362_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") monotonic
        ret <4 x i8> %retval
}

define <8 x i8> @nand_monotonic_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: nand_monotonic_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_monotonic_v8i8_global_cta_param_0];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:  $L__BB363_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r8;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r3, %r8, %r2;
; SM70-NEXT:    and.b32 %r4, %r7, %r1;
; SM70-NEXT:    xor.b32 %r5, %r4, -1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    xor.b32 %r6, %r3, -1;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB363_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") monotonic
        ret <8 x i8> %retval
}

define <1 x i8> @nand_acquire_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: nand_acquire_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<16>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [nand_acquire_v1i8_global_cta_param_0];
; SM70-NEXT:    ld.param.b8 %r6, [nand_acquire_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:    and.b32 %r8, %r7, 3;
; SM70-NEXT:    shl.b32 %r1, %r8, 3;
; SM70-NEXT:    shl.b32 %r2, 255, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    shl.b32 %r4, %r6, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT:  $L__BB364_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r9, %r15, %r4;
; SM70-NEXT:    not.b32 %r10, %r9;
; SM70-NEXT:    and.b32 %r11, %r10, %r2;
; SM70-NEXT:    and.b32 %r12, %r15, %r3;
; SM70-NEXT:    or.b32 %r13, %r12, %r11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r15;
; SM70-NEXT:    mov.b32 %r15, %r5;
; SM70-NEXT:    @%p1 bra $L__BB364_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r14, %r5, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b8 [func_retval0], %r14;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") acquire
        ret <1 x i8> %retval
}

define <2 x i8> @nand_acquire_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: nand_acquire_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<16>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [nand_acquire_v2i8_global_cta_param_0];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r6, %rd2;
; SM70-NEXT:    and.b32 %r7, %r6, 3;
; SM70-NEXT:    shl.b32 %r1, %r7, 3;
; SM70-NEXT:    shl.b32 %r2, 65535, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    ld.param.b16 %r8, [nand_acquire_v2i8_global_cta_param_1];
; SM70-NEXT:    shl.b32 %r4, %r8, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT:  $L__BB365_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r9, %r15, %r4;
; SM70-NEXT:    not.b32 %r10, %r9;
; SM70-NEXT:    and.b32 %r11, %r10, %r2;
; SM70-NEXT:    and.b32 %r12, %r15, %r3;
; SM70-NEXT:    or.b32 %r13, %r12, %r11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r15;
; SM70-NEXT:    mov.b32 %r15, %r5;
; SM70-NEXT:    @%p1 bra $L__BB365_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r14, %r5, %r1;
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b16 [func_retval0], %r14;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") acquire
        ret <2 x i8> %retval
}

define <4 x i8> @nand_acquire_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: nand_acquire_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<6>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [nand_acquire_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acquire_v4i8_global_cta_param_0];
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT:  $L__BB366_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r3, %r5, %r2;
; SM70-NEXT:    xor.b32 %r4, %r3, -1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r5;
; SM70-NEXT:    mov.b32 %r5, %r1;
; SM70-NEXT:    @%p1 bra $L__BB366_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") acquire
        ret <4 x i8> %retval
}

define <8 x i8> @nand_acquire_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: nand_acquire_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acquire_v8i8_global_cta_param_0];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:  $L__BB367_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r8;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r3, %r8, %r2;
; SM70-NEXT:    and.b32 %r4, %r7, %r1;
; SM70-NEXT:    xor.b32 %r5, %r4, -1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    xor.b32 %r6, %r3, -1;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB367_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") acquire
        ret <8 x i8> %retval
}

define <1 x i8> @nand_release_v1i8_global_cta(ptr addrspace(1) %addr, <1 x i8> %val) {
; SM70-LABEL: nand_release_v1i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<16>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [nand_release_v1i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b8 %r6, [nand_release_v1i8_global_cta_param_1];
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:    and.b32 %r8, %r7, 3;
; SM70-NEXT:    shl.b32 %r1, %r8, 3;
; SM70-NEXT:    shl.b32 %r2, 255, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    shl.b32 %r4, %r6, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT:  $L__BB368_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r9, %r15, %r4;
; SM70-NEXT:    not.b32 %r10, %r9;
; SM70-NEXT:    and.b32 %r11, %r10, %r2;
; SM70-NEXT:    and.b32 %r12, %r15, %r3;
; SM70-NEXT:    or.b32 %r13, %r12, %r11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r15;
; SM70-NEXT:    mov.b32 %r15, %r5;
; SM70-NEXT:    @%p1 bra $L__BB368_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r14, %r5, %r1;
; SM70-NEXT:    st.param.b8 [func_retval0], %r14;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <1 x i8> %val syncscope("block") release
        ret <1 x i8> %retval
}

define <2 x i8> @nand_release_v2i8_global_cta(ptr addrspace(1) %addr, <2 x i8> %val) {
; SM70-LABEL: nand_release_v2i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<16>;
; SM70-NEXT:    .reg .b64 %rd<3>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd2, [nand_release_v2i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
; SM70-NEXT:    cvt.u32.u64 %r6, %rd2;
; SM70-NEXT:    and.b32 %r7, %r6, 3;
; SM70-NEXT:    shl.b32 %r1, %r7, 3;
; SM70-NEXT:    shl.b32 %r2, 65535, %r1;
; SM70-NEXT:    not.b32 %r3, %r2;
; SM70-NEXT:    ld.param.b16 %r8, [nand_release_v2i8_global_cta_param_1];
; SM70-NEXT:    shl.b32 %r4, %r8, %r1;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r15, [%rd1];
; SM70-NEXT:  $L__BB369_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r9, %r15, %r4;
; SM70-NEXT:    not.b32 %r10, %r9;
; SM70-NEXT:    and.b32 %r11, %r10, %r2;
; SM70-NEXT:    and.b32 %r12, %r15, %r3;
; SM70-NEXT:    or.b32 %r13, %r12, %r11;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13;
; SM70-NEXT:    setp.ne.b32 %p1, %r5, %r15;
; SM70-NEXT:    mov.b32 %r15, %r5;
; SM70-NEXT:    @%p1 bra $L__BB369_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    shr.u32 %r14, %r5, %r1;
; SM70-NEXT:    st.param.b16 [func_retval0], %r14;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <2 x i8> %val syncscope("block") release
        ret <2 x i8> %retval
}

define <4 x i8> @nand_release_v4i8_global_cta(ptr addrspace(1) %addr, <4 x i8> %val) {
; SM70-LABEL: nand_release_v4i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<6>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b32 %r2, [nand_release_v4i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_release_v4i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT:  $L__BB370_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r3, %r5, %r2;
; SM70-NEXT:    xor.b32 %r4, %r3, -1;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4;
; SM70-NEXT:    setp.ne.b32 %p1, %r1, %r5;
; SM70-NEXT:    mov.b32 %r5, %r1;
; SM70-NEXT:    @%p1 bra $L__BB370_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    st.param.b32 [func_retval0], %r1;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <4 x i8> %val syncscope("block") release
        ret <4 x i8> %retval
}

define <8 x i8> @nand_release_v8i8_global_cta(ptr addrspace(1) %addr, <8 x i8> %val) {
; SM70-LABEL: nand_release_v8i8_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<9>;
; SM70-NEXT:    .reg .b64 %rd<12>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [nand_release_v8i8_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_release_v8i8_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd2;
; SM70-NEXT:  $L__BB371_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r8;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r3, %r8, %r2;
; SM70-NEXT:    and.b32 %r4, %r7, %r1;
; SM70-NEXT:    xor.b32 %r5, %r4, -1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r5;
; SM70-NEXT:    xor.b32 %r6, %r3, -1;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r6;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd11, [%rd1], %rd6, %rd10;
; SM70-NEXT:    setp.ne.b64 %p1, %rd11, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r8}, %rd11; }
; SM70-NEXT:    cvt.u32.u64 %r7, %rd11;
; SM70-NEXT:    @%p1 bra $L__BB371_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r7, %r8};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <8 x i8> %val syncscope("block") release
        ret <8 x i8> %retval
}

define <1 x i32> @nand_monotonic_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: nand_monotonic_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<6>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [nand_monotonic_v1i32_global_cta_param_0];
; SM70-NEXT:    ld.param.b32 %r1, [nand_monotonic_v1i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT:  $L__BB372_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r3, %r5, %r1;
; SM70-NEXT:    not.b32 %r4, %r3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM70-NEXT:    setp.ne.b32 %p1, %r2, %r5;
; SM70-NEXT:    mov.b32 %r5, %r2;
; SM70-NEXT:    @%p1 bra $L__BB372_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") monotonic
        ret <1 x i32> %retval
}

define <2 x i32> @nand_monotonic_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: nand_monotonic_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<13>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [nand_monotonic_v2i32_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_monotonic_v2i32_global_cta_param_0];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:  $L__BB373_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r3, %r6, %r2;
; SM70-NEXT:    and.b32 %r4, %r5, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r4;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r3;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    not.b64 %rd11, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd12;
; SM70-NEXT:    @%p1 bra $L__BB373_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") monotonic
        ret <2 x i32> %retval
}

define <4 x i32> @nand_monotonic_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: nand_monotonic_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<24>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [nand_monotonic_v4i32_global_cta_param_0];
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v4i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:  $L__BB374_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r5, %r10, %r2;
; SM70-NEXT:    and.b32 %r6, %r9, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r6;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r5;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    not.b64 %rd11, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd12;
; SM70-NEXT:    @%p1 bra $L__BB374_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd13;
; SM70-NEXT:  $L__BB374_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd14, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd15, %r12;
; SM70-NEXT:    shl.b64 %rd16, %rd15, 32;
; SM70-NEXT:    or.b64 %rd17, %rd14, %rd16;
; SM70-NEXT:    and.b32 %r7, %r12, %r4;
; SM70-NEXT:    and.b32 %r8, %r11, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r8;
; SM70-NEXT:    cvt.u64.u32 %rd19, %r7;
; SM70-NEXT:    shl.b64 %rd20, %rd19, 32;
; SM70-NEXT:    or.b64 %rd21, %rd18, %rd20;
; SM70-NEXT:    not.b64 %rd22, %rd21;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
; SM70-NEXT:    setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd23;
; SM70-NEXT:    @%p2 bra $L__BB374_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") monotonic
        ret <4 x i32> %retval
}

define <8 x i32> @nand_monotonic_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: nand_monotonic_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<5>;
; SM70-NEXT:    .reg .b32 %r<25>;
; SM70-NEXT:    .reg .b64 %rd<46>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [nand_monotonic_v8i32_global_cta_param_0];
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_monotonic_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_monotonic_v8i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NEXT:  $L__BB375_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r17;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r18;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r9, %r18, %r6;
; SM70-NEXT:    and.b32 %r10, %r17, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r10;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r9;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    not.b64 %rd11, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r17, %rd12;
; SM70-NEXT:    @%p1 bra $L__BB375_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM70-NEXT:    cvt.u32.u64 %r19, %rd13;
; SM70-NEXT:  $L__BB375_3: // %atomicrmw.start6
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd14, %r19;
; SM70-NEXT:    cvt.u64.u32 %rd15, %r20;
; SM70-NEXT:    shl.b64 %rd16, %rd15, 32;
; SM70-NEXT:    or.b64 %rd17, %rd14, %rd16;
; SM70-NEXT:    and.b32 %r11, %r20, %r8;
; SM70-NEXT:    and.b32 %r12, %r19, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NEXT:    cvt.u64.u32 %rd19, %r11;
; SM70-NEXT:    shl.b64 %rd20, %rd19, 32;
; SM70-NEXT:    or.b64 %rd21, %rd18, %rd20;
; SM70-NEXT:    not.b64 %rd22, %rd21;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
; SM70-NEXT:    setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM70-NEXT:    cvt.u32.u64 %r19, %rd23;
; SM70-NEXT:    @%p2 bra $L__BB375_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM70-NEXT:    cvt.u32.u64 %r21, %rd24;
; SM70-NEXT:  $L__BB375_5: // %atomicrmw.start16
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd25, %r21;
; SM70-NEXT:    cvt.u64.u32 %rd26, %r22;
; SM70-NEXT:    shl.b64 %rd27, %rd26, 32;
; SM70-NEXT:    or.b64 %rd28, %rd25, %rd27;
; SM70-NEXT:    and.b32 %r13, %r22, %r2;
; SM70-NEXT:    and.b32 %r14, %r21, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd29, %r14;
; SM70-NEXT:    cvt.u64.u32 %rd30, %r13;
; SM70-NEXT:    shl.b64 %rd31, %rd30, 32;
; SM70-NEXT:    or.b64 %rd32, %rd29, %rd31;
; SM70-NEXT:    not.b64 %rd33, %rd32;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
; SM70-NEXT:    setp.ne.b64 %p3, %rd34, %rd28;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM70-NEXT:    cvt.u32.u64 %r21, %rd34;
; SM70-NEXT:    @%p3 bra $L__BB375_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end15
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM70-NEXT:    cvt.u32.u64 %r23, %rd35;
; SM70-NEXT:  $L__BB375_7: // %atomicrmw.start22
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd36, %r23;
; SM70-NEXT:    cvt.u64.u32 %rd37, %r24;
; SM70-NEXT:    shl.b64 %rd38, %rd37, 32;
; SM70-NEXT:    or.b64 %rd39, %rd36, %rd38;
; SM70-NEXT:    and.b32 %r15, %r24, %r4;
; SM70-NEXT:    and.b32 %r16, %r23, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd40, %r16;
; SM70-NEXT:    cvt.u64.u32 %rd41, %r15;
; SM70-NEXT:    shl.b64 %rd42, %rd41, 32;
; SM70-NEXT:    or.b64 %rd43, %rd40, %rd42;
; SM70-NEXT:    not.b64 %rd44, %rd43;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
; SM70-NEXT:    setp.ne.b64 %p4, %rd45, %rd39;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM70-NEXT:    cvt.u32.u64 %r23, %rd45;
; SM70-NEXT:    @%p4 bra $L__BB375_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end21
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") monotonic
        ret <8 x i32> %retval
}

define <1 x i32> @nand_acquire_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: nand_acquire_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<6>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acquire_v1i32_global_cta_param_0];
; SM70-NEXT:    ld.param.b32 %r1, [nand_acquire_v1i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT:  $L__BB376_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r3, %r5, %r1;
; SM70-NEXT:    not.b32 %r4, %r3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM70-NEXT:    setp.ne.b32 %p1, %r2, %r5;
; SM70-NEXT:    mov.b32 %r5, %r2;
; SM70-NEXT:    @%p1 bra $L__BB376_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") acquire
        ret <1 x i32> %retval
}

define <2 x i32> @nand_acquire_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: nand_acquire_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<13>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [nand_acquire_v2i32_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acquire_v2i32_global_cta_param_0];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:  $L__BB377_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r3, %r6, %r2;
; SM70-NEXT:    and.b32 %r4, %r5, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r4;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r3;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    not.b64 %rd11, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd12;
; SM70-NEXT:    @%p1 bra $L__BB377_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") acquire
        ret <2 x i32> %retval
}

define <4 x i32> @nand_acquire_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: nand_acquire_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<24>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acquire_v4i32_global_cta_param_0];
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v4i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:  $L__BB378_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r5, %r10, %r2;
; SM70-NEXT:    and.b32 %r6, %r9, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r6;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r5;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    not.b64 %rd11, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd12;
; SM70-NEXT:    @%p1 bra $L__BB378_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd13;
; SM70-NEXT:  $L__BB378_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd14, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd15, %r12;
; SM70-NEXT:    shl.b64 %rd16, %rd15, 32;
; SM70-NEXT:    or.b64 %rd17, %rd14, %rd16;
; SM70-NEXT:    and.b32 %r7, %r12, %r4;
; SM70-NEXT:    and.b32 %r8, %r11, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r8;
; SM70-NEXT:    cvt.u64.u32 %rd19, %r7;
; SM70-NEXT:    shl.b64 %rd20, %rd19, 32;
; SM70-NEXT:    or.b64 %rd21, %rd18, %rd20;
; SM70-NEXT:    not.b64 %rd22, %rd21;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
; SM70-NEXT:    setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd23;
; SM70-NEXT:    @%p2 bra $L__BB378_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") acquire
        ret <4 x i32> %retval
}

define <8 x i32> @nand_acquire_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: nand_acquire_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<5>;
; SM70-NEXT:    .reg .b32 %r<25>;
; SM70-NEXT:    .reg .b64 %rd<46>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [nand_acquire_v8i32_global_cta_param_0];
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_acquire_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_acquire_v8i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NEXT:  $L__BB379_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r17;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r18;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r9, %r18, %r6;
; SM70-NEXT:    and.b32 %r10, %r17, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r10;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r9;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    not.b64 %rd11, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r17, %rd12;
; SM70-NEXT:    @%p1 bra $L__BB379_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM70-NEXT:    cvt.u32.u64 %r19, %rd13;
; SM70-NEXT:  $L__BB379_3: // %atomicrmw.start6
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd14, %r19;
; SM70-NEXT:    cvt.u64.u32 %rd15, %r20;
; SM70-NEXT:    shl.b64 %rd16, %rd15, 32;
; SM70-NEXT:    or.b64 %rd17, %rd14, %rd16;
; SM70-NEXT:    and.b32 %r11, %r20, %r8;
; SM70-NEXT:    and.b32 %r12, %r19, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NEXT:    cvt.u64.u32 %rd19, %r11;
; SM70-NEXT:    shl.b64 %rd20, %rd19, 32;
; SM70-NEXT:    or.b64 %rd21, %rd18, %rd20;
; SM70-NEXT:    not.b64 %rd22, %rd21;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
; SM70-NEXT:    setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM70-NEXT:    cvt.u32.u64 %r19, %rd23;
; SM70-NEXT:    @%p2 bra $L__BB379_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM70-NEXT:    cvt.u32.u64 %r21, %rd24;
; SM70-NEXT:  $L__BB379_5: // %atomicrmw.start16
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd25, %r21;
; SM70-NEXT:    cvt.u64.u32 %rd26, %r22;
; SM70-NEXT:    shl.b64 %rd27, %rd26, 32;
; SM70-NEXT:    or.b64 %rd28, %rd25, %rd27;
; SM70-NEXT:    and.b32 %r13, %r22, %r2;
; SM70-NEXT:    and.b32 %r14, %r21, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd29, %r14;
; SM70-NEXT:    cvt.u64.u32 %rd30, %r13;
; SM70-NEXT:    shl.b64 %rd31, %rd30, 32;
; SM70-NEXT:    or.b64 %rd32, %rd29, %rd31;
; SM70-NEXT:    not.b64 %rd33, %rd32;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
; SM70-NEXT:    setp.ne.b64 %p3, %rd34, %rd28;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM70-NEXT:    cvt.u32.u64 %r21, %rd34;
; SM70-NEXT:    @%p3 bra $L__BB379_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end15
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM70-NEXT:    cvt.u32.u64 %r23, %rd35;
; SM70-NEXT:  $L__BB379_7: // %atomicrmw.start22
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd36, %r23;
; SM70-NEXT:    cvt.u64.u32 %rd37, %r24;
; SM70-NEXT:    shl.b64 %rd38, %rd37, 32;
; SM70-NEXT:    or.b64 %rd39, %rd36, %rd38;
; SM70-NEXT:    and.b32 %r15, %r24, %r4;
; SM70-NEXT:    and.b32 %r16, %r23, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd40, %r16;
; SM70-NEXT:    cvt.u64.u32 %rd41, %r15;
; SM70-NEXT:    shl.b64 %rd42, %rd41, 32;
; SM70-NEXT:    or.b64 %rd43, %rd40, %rd42;
; SM70-NEXT:    not.b64 %rd44, %rd43;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
; SM70-NEXT:    setp.ne.b64 %p4, %rd45, %rd39;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM70-NEXT:    cvt.u32.u64 %r23, %rd45;
; SM70-NEXT:    @%p4 bra $L__BB379_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end21
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") acquire
        ret <8 x i32> %retval
}

define <1 x i32> @nand_release_v1i32_global_cta(ptr addrspace(1) %addr, <1 x i32> %val) {
; SM70-LABEL: nand_release_v1i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<6>;
; SM70-NEXT:    .reg .b64 %rd<2>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [nand_release_v1i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.b32 %r1, [nand_release_v1i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b32 %r5, [%rd1];
; SM70-NEXT:  $L__BB380_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    and.b32 %r3, %r5, %r1;
; SM70-NEXT:    not.b32 %r4, %r3;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r2, [%rd1], %r5, %r4;
; SM70-NEXT:    setp.ne.b32 %p1, %r2, %r5;
; SM70-NEXT:    mov.b32 %r5, %r2;
; SM70-NEXT:    @%p1 bra $L__BB380_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    st.param.b32 [func_retval0], %r2;
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <1 x i32> %val syncscope("block") release
        ret <1 x i32> %retval
}

define <2 x i32> @nand_release_v2i32_global_cta(ptr addrspace(1) %addr, <2 x i32> %val) {
; SM70-LABEL: nand_release_v2i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<2>;
; SM70-NEXT:    .reg .b32 %r<7>;
; SM70-NEXT:    .reg .b64 %rd<13>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.v2.b32 {%r1, %r2}, [nand_release_v2i32_global_cta_param_1];
; SM70-NEXT:    ld.param.b64 %rd1, [nand_release_v2i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd2;
; SM70-NEXT:  $L__BB381_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r6;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r3, %r6, %r2;
; SM70-NEXT:    and.b32 %r4, %r5, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r4;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r3;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    not.b64 %rd11, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r6}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r5, %rd12;
; SM70-NEXT:    @%p1 bra $L__BB381_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <2 x i32> %val syncscope("block") release
        ret <2 x i32> %retval
}

define <4 x i32> @nand_release_v4i32_global_cta(ptr addrspace(1) %addr, <4 x i32> %val) {
; SM70-LABEL: nand_release_v4i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<3>;
; SM70-NEXT:    .reg .b32 %r<13>;
; SM70-NEXT:    .reg .b64 %rd<24>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [nand_release_v4i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v4i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
; SM70-NEXT:  $L__BB382_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r9;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r10;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r5, %r10, %r2;
; SM70-NEXT:    and.b32 %r6, %r9, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r6;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r5;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    not.b64 %rd11, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r10}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r9, %rd12;
; SM70-NEXT:    @%p1 bra $L__BB382_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd13; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd13;
; SM70-NEXT:  $L__BB382_3: // %atomicrmw.start2
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd14, %r11;
; SM70-NEXT:    cvt.u64.u32 %rd15, %r12;
; SM70-NEXT:    shl.b64 %rd16, %rd15, 32;
; SM70-NEXT:    or.b64 %rd17, %rd14, %rd16;
; SM70-NEXT:    and.b32 %r7, %r12, %r4;
; SM70-NEXT:    and.b32 %r8, %r11, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r8;
; SM70-NEXT:    cvt.u64.u32 %rd19, %r7;
; SM70-NEXT:    shl.b64 %rd20, %rd19, 32;
; SM70-NEXT:    or.b64 %rd21, %rd18, %rd20;
; SM70-NEXT:    not.b64 %rd22, %rd21;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
; SM70-NEXT:    setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r12}, %rd23; }
; SM70-NEXT:    cvt.u32.u64 %r11, %rd23;
; SM70-NEXT:    @%p2 bra $L__BB382_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end1
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r9, %r10, %r11, %r12};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <4 x i32> %val syncscope("block") release
        ret <4 x i32> %retval
}

define <8 x i32> @nand_release_v8i32_global_cta(ptr addrspace(1) %addr, <8 x i32> %val) {
; SM70-LABEL: nand_release_v8i32_global_cta(
; SM70:       {
; SM70-NEXT:    .reg .pred %p<5>;
; SM70-NEXT:    .reg .b32 %r<25>;
; SM70-NEXT:    .reg .b64 %rd<46>;
; SM70-EMPTY:
; SM70-NEXT:  // %bb.0:
; SM70-NEXT:    ld.param.b64 %rd1, [nand_release_v8i32_global_cta_param_0];
; SM70-NEXT:    fence.acq_rel.cta;
; SM70-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [nand_release_v8i32_global_cta_param_1+16];
; SM70-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [nand_release_v8i32_global_cta_param_1];
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd2, [%rd1];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd2; }
; SM70-NEXT:    cvt.u32.u64 %r17, %rd2;
; SM70-NEXT:  $L__BB383_1: // %atomicrmw.start
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd3, %r17;
; SM70-NEXT:    cvt.u64.u32 %rd4, %r18;
; SM70-NEXT:    shl.b64 %rd5, %rd4, 32;
; SM70-NEXT:    or.b64 %rd6, %rd3, %rd5;
; SM70-NEXT:    and.b32 %r9, %r18, %r6;
; SM70-NEXT:    and.b32 %r10, %r17, %r5;
; SM70-NEXT:    cvt.u64.u32 %rd7, %r10;
; SM70-NEXT:    cvt.u64.u32 %rd8, %r9;
; SM70-NEXT:    shl.b64 %rd9, %rd8, 32;
; SM70-NEXT:    or.b64 %rd10, %rd7, %rd9;
; SM70-NEXT:    not.b64 %rd11, %rd10;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd12, [%rd1], %rd6, %rd11;
; SM70-NEXT:    setp.ne.b64 %p1, %rd12, %rd6;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r18}, %rd12; }
; SM70-NEXT:    cvt.u32.u64 %r17, %rd12;
; SM70-NEXT:    @%p1 bra $L__BB383_1;
; SM70-NEXT:  // %bb.2: // %atomicrmw.end
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd13, [%rd1+8];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd13; }
; SM70-NEXT:    cvt.u32.u64 %r19, %rd13;
; SM70-NEXT:  $L__BB383_3: // %atomicrmw.start6
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd14, %r19;
; SM70-NEXT:    cvt.u64.u32 %rd15, %r20;
; SM70-NEXT:    shl.b64 %rd16, %rd15, 32;
; SM70-NEXT:    or.b64 %rd17, %rd14, %rd16;
; SM70-NEXT:    and.b32 %r11, %r20, %r8;
; SM70-NEXT:    and.b32 %r12, %r19, %r7;
; SM70-NEXT:    cvt.u64.u32 %rd18, %r12;
; SM70-NEXT:    cvt.u64.u32 %rd19, %r11;
; SM70-NEXT:    shl.b64 %rd20, %rd19, 32;
; SM70-NEXT:    or.b64 %rd21, %rd18, %rd20;
; SM70-NEXT:    not.b64 %rd22, %rd21;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd23, [%rd1+8], %rd17, %rd22;
; SM70-NEXT:    setp.ne.b64 %p2, %rd23, %rd17;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r20}, %rd23; }
; SM70-NEXT:    cvt.u32.u64 %r19, %rd23;
; SM70-NEXT:    @%p2 bra $L__BB383_3;
; SM70-NEXT:  // %bb.4: // %atomicrmw.end5
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd24, [%rd1+16];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd24; }
; SM70-NEXT:    cvt.u32.u64 %r21, %rd24;
; SM70-NEXT:  $L__BB383_5: // %atomicrmw.start16
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd25, %r21;
; SM70-NEXT:    cvt.u64.u32 %rd26, %r22;
; SM70-NEXT:    shl.b64 %rd27, %rd26, 32;
; SM70-NEXT:    or.b64 %rd28, %rd25, %rd27;
; SM70-NEXT:    and.b32 %r13, %r22, %r2;
; SM70-NEXT:    and.b32 %r14, %r21, %r1;
; SM70-NEXT:    cvt.u64.u32 %rd29, %r14;
; SM70-NEXT:    cvt.u64.u32 %rd30, %r13;
; SM70-NEXT:    shl.b64 %rd31, %rd30, 32;
; SM70-NEXT:    or.b64 %rd32, %rd29, %rd31;
; SM70-NEXT:    not.b64 %rd33, %rd32;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd34, [%rd1+16], %rd28, %rd33;
; SM70-NEXT:    setp.ne.b64 %p3, %rd34, %rd28;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r22}, %rd34; }
; SM70-NEXT:    cvt.u32.u64 %r21, %rd34;
; SM70-NEXT:    @%p3 bra $L__BB383_5;
; SM70-NEXT:  // %bb.6: // %atomicrmw.end15
; SM70-NEXT:    ld.relaxed.cta.global.b64 %rd35, [%rd1+24];
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd35; }
; SM70-NEXT:    cvt.u32.u64 %r23, %rd35;
; SM70-NEXT:  $L__BB383_7: // %atomicrmw.start22
; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
; SM70-NEXT:    cvt.u64.u32 %rd36, %r23;
; SM70-NEXT:    cvt.u64.u32 %rd37, %r24;
; SM70-NEXT:    shl.b64 %rd38, %rd37, 32;
; SM70-NEXT:    or.b64 %rd39, %rd36, %rd38;
; SM70-NEXT:    and.b32 %r15, %r24, %r4;
; SM70-NEXT:    and.b32 %r16, %r23, %r3;
; SM70-NEXT:    cvt.u64.u32 %rd40, %r16;
; SM70-NEXT:    cvt.u64.u32 %rd41, %r15;
; SM70-NEXT:    shl.b64 %rd42, %rd41, 32;
; SM70-NEXT:    or.b64 %rd43, %rd40, %rd42;
; SM70-NEXT:    not.b64 %rd44, %rd43;
; SM70-NEXT:    atom.relaxed.cta.global.cas.b64 %rd45, [%rd1+24], %rd39, %rd44;
; SM70-NEXT:    setp.ne.b64 %p4, %rd45, %rd39;
; SM70-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r24}, %rd45; }
; SM70-NEXT:    cvt.u32.u64 %r23, %rd45;
; SM70-NEXT:    @%p4 bra $L__BB383_7;
; SM70-NEXT:  // %bb.8: // %atomicrmw.end21
; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r21, %r22, %r23, %r24};
; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r17, %r18, %r19, %r20};
; SM70-NEXT:    ret;
        %retval = atomicrmw elementwise nand ptr  addrspace(1) %addr, <8 x i32> %val syncscope("block") release
        ret <8 x i32> %retval
}