| # For manual usage, not as a part of lit tests. Used for generating the following tests: |
| |
| from string import Template |
| from itertools import product |
| |
| TEST_SM_ARCH_PAIRS = [(60, 50), (70, 63), (90, 87)] |
| |
| SCOPE_LLVM_TO_PTX = {"": "sys", "block": "cta", "cluster": "cluster", "device": "gpu"} |
| |
| ORDERINGS = ["monotonic", "acquire", "release", "acq_rel", "seq_cst"] |
| |
| INTEGER_OPERATIONS = [ |
| "xchg", |
| "add", |
| "sub", |
| "and", |
| "nand", |
| "or", |
| "xor", |
| "max", |
| "min", |
| "umax", |
| "umin", |
| "uinc_wrap", |
| "udec_wrap", |
| "usub_cond", |
| "usub_sat", |
| ] |
| |
| FLOATING_POINT_OPERATIONS = ["fadd", "fsub", "fmin", "fmax", "fminimum", "fmaximum"] |
| |
| ADDRSPACE_NUM_TO_ADDRSPACE = {0: "generic", 1: "global", 3: "shared"} |
| |
| # A small spread of operations used to exercise the scope and address-space |
| # qualifiers, which are emitted orthogonally to the operation: a native integer op |
| # at two sizes, signed/unsigned variants, an always-emulated op (lowers to a cas |
| # loop), and floating-point add. |
| REPRESENTATIVE_OPS = [ |
| ("add", "i32"), |
| ("add", "i64"), |
| ("min", "i32"), |
| ("umax", "i32"), |
| ("nand", "i32"), |
| ("nand", "i64"), |
| ("fadd", "float"), |
| ("fadd", "double"), |
| ] |
| |
| atomicrmw_func = Template( |
| """define ${datatype} @${operation}_${ordering}_${datatype}_${addrspace}_${ptx_scope}(ptr${addrspace_cast} %addr, ${datatype} %val) { |
| %retval = atomicrmw ${operation} ptr ${addrspace_cast} %addr, ${datatype} %val syncscope(\"${llvm_scope}\") ${ordering} |
| ret $datatype %retval |
| } |
| """ |
| ) |
| |
| # atomicrmw fadd's lowering depends on the function's FTZ (denormal) mode, so we |
| # check codegen both with and without it. Lines common to both runs collapse to |
| # the SM${sm} prefix; only the FTZ-sensitive ops diverge into SM${sm}-NOFTZ / |
| # SM${sm}-FTZ. (-nvptx-allow-ftz-atomics is covered separately in |
| # atomicrmw-allow-ftz-atomics.ll.) |
| run_statement = Template( |
| """; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ-DISALLOW |
| ; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ-ALLOW |
| ; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ-ALLOW |
| ; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ-DISALLOW |
| ; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ-ALLOW |
| ; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ-ALLOW |
| ; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_${sm} %} |
| ; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_${sm} %} |
| ; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | %ptxas-verify -arch=sm_${sm} %} |
| ; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=false | %ptxas-verify -arch=sm_${sm} %} |
| ; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign -nvptx-allow-ftz-atomics=true | %ptxas-verify -arch=sm_${sm} %} |
| ; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_${sm} %} |
| """ |
| ) |
| |
| |
| def get_addrspace_cast(addrspace): |
| if addrspace == 0: |
| return "" |
| else: |
| return " addrspace({})".format(str(addrspace)) |
| |
| |
| if __name__ == "__main__": |
| for sm, ptx in TEST_SM_ARCH_PAIRS: |
| # Slice 1: Keep addrspace, llvm_scope, ordering fixed, generate all possible operations and sizes |
| with open("atomicrmw-sm{}.ll".format(str(sm)), "w") as fp: |
| print(run_statement.substitute(sm=sm, ptx=ptx, ptxfp=ptx / 10.0), file=fp) |
| # Integer operations |
| addrspace, llvm_scope, ordering = 1, "block", "acq_rel" |
| for operation, datatype in product( |
| INTEGER_OPERATIONS, ["i8", "i16", "i32", "i64"] |
| ): |
| print( |
| atomicrmw_func.substitute( |
| operation=operation, |
| ordering=ordering, |
| datatype=datatype, |
| addrspace=ADDRSPACE_NUM_TO_ADDRSPACE[addrspace], |
| ptx_scope=SCOPE_LLVM_TO_PTX[llvm_scope], |
| llvm_scope=llvm_scope, |
| addrspace_cast=get_addrspace_cast(addrspace), |
| ), |
| file=fp, |
| ) |
| |
| # Floating point add |
| for datatype, operation in product( |
| ["float", "double", "half", "bfloat"], FLOATING_POINT_OPERATIONS |
| ): |
| print( |
| atomicrmw_func.substitute( |
| operation=operation, |
| ordering=ordering, |
| datatype=datatype, |
| addrspace=ADDRSPACE_NUM_TO_ADDRSPACE[addrspace], |
| ptx_scope=SCOPE_LLVM_TO_PTX[llvm_scope], |
| llvm_scope=llvm_scope, |
| addrspace_cast=get_addrspace_cast(addrspace), |
| ), |
| file=fp, |
| ) |
| |
| # Slice 2: Keep addrspace, llvm_scope fixed, and generate all possible orderings for operations add and nand. |
| # add is natively supported for larger bitwidths, while nand is emulated always |
| addrspace, llvm_scope = 1, "block" |
| for operation, datatype, ordering in product( |
| ["add", "nand"], ["i8", "i32"], ORDERINGS |
| ): |
| if addrspace == 1 and llvm_scope == "block" and ordering == "acq_rel": |
| # These are a part of Slice 1 |
| continue |
| print( |
| atomicrmw_func.substitute( |
| operation=operation, |
| ordering=ordering, |
| datatype=datatype, |
| addrspace=ADDRSPACE_NUM_TO_ADDRSPACE[addrspace], |
| addrspace_cast=get_addrspace_cast(addrspace), |
| ptx_scope=SCOPE_LLVM_TO_PTX[llvm_scope], |
| llvm_scope=llvm_scope, |
| ), |
| file=fp, |
| ) |
| |
| # Slice 3: Keep addrspace (global) and ordering fixed, vary the scope |
| # qualifier. block/global is already covered by Slice 1. |
| addrspace, ordering = 1, "acq_rel" |
| for llvm_scope in [s for s in SCOPE_LLVM_TO_PTX if s != "block"]: |
| for operation, datatype in REPRESENTATIVE_OPS: |
| print( |
| atomicrmw_func.substitute( |
| operation=operation, |
| ordering=ordering, |
| datatype=datatype, |
| addrspace=ADDRSPACE_NUM_TO_ADDRSPACE[addrspace], |
| addrspace_cast=get_addrspace_cast(addrspace), |
| ptx_scope=SCOPE_LLVM_TO_PTX[llvm_scope], |
| llvm_scope=llvm_scope, |
| ), |
| file=fp, |
| ) |
| |
| # Slice 4: Keep scope (block) and ordering fixed, vary the address-space |
| # qualifier. global is already covered by Slice 1. |
| llvm_scope, ordering = "block", "acq_rel" |
| for addrspace in [a for a in ADDRSPACE_NUM_TO_ADDRSPACE if a != 1]: |
| for operation, datatype in REPRESENTATIVE_OPS: |
| print( |
| atomicrmw_func.substitute( |
| operation=operation, |
| ordering=ordering, |
| datatype=datatype, |
| addrspace=ADDRSPACE_NUM_TO_ADDRSPACE[addrspace], |
| addrspace_cast=get_addrspace_cast(addrspace), |
| ptx_scope=SCOPE_LLVM_TO_PTX[llvm_scope], |
| llvm_scope=llvm_scope, |
| ), |
| file=fp, |
| ) |
