libclc: Add canonicalize utility functions (#187357) This is mostly to work around spirv's canonicalize still being broken. GitOrigin-RevId: 85e9ac589819b0f24074e92dcaee5cc05446064e
diff --git a/clc/include/clc/math/clc_canonicalize.h b/clc/include/clc/math/clc_canonicalize.h new file mode 100644 index 0000000..527463e --- /dev/null +++ b/clc/include/clc/math/clc_canonicalize.h
@@ -0,0 +1,21 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_MATH_CLC_CANONICALIZE_H__ +#define __CLC_MATH_CLC_CANONICALIZE_H__ + +#include "clc/internal/clc.h" + +#define __CLC_BODY "clc/math/unary_decl.inc" +#define __CLC_FUNCTION __clc_canonicalize + +#include "clc/math/gentype.inc" + +#undef __CLC_FUNCTION + +#endif // __CLC_MATH_CLC_CANONICALIZE_H__
diff --git a/clc/include/clc/math/clc_flush_if_daz.h b/clc/include/clc/math/clc_flush_if_daz.h new file mode 100644 index 0000000..9e9bf99 --- /dev/null +++ b/clc/include/clc/math/clc_flush_if_daz.h
@@ -0,0 +1,21 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_MATH_CLC_FLUSH_IF_DAZ_H__ +#define __CLC_MATH_CLC_FLUSH_IF_DAZ_H__ + +#include "clc/internal/clc.h" + +#define __CLC_BODY "clc/math/unary_decl.inc" +#define __CLC_FUNCTION __clc_flush_if_daz + +#include "clc/math/gentype.inc" + +#undef __CLC_FUNCTION + +#endif // __CLC_MATH_CLC_FLUSH_IF_DAZ_H__
diff --git a/clc/include/clc/math/math.h b/clc/include/clc/math/math.h index 15b1272..a676c35 100644 --- a/clc/include/clc/math/math.h +++ b/clc/include/clc/math/math.h
@@ -58,16 +58,6 @@ #define LOG_MAGIC_NUM_SP32 (1 + NUMEXPBITS_SP32 - EXPBIAS_SP32) -_CLC_OVERLOAD _CLC_INLINE float __clc_flush_denormal_if_not_supported(float x) { - int ix = __clc_as_int(x); - if (__clc_denormals_are_zero_fp32() && ((ix & EXPBITS_SP32) == 0) && - ((ix & MANTBITS_SP32) != 0)) { - ix &= SIGNBIT_SP32; - x = __clc_as_float(ix); - } - return x; -} - #ifdef cl_khr_fp64 #define SIGNBIT_DP64 0x8000000000000000L
diff --git a/clc/lib/clspv/math/clc_sw_fma.cl b/clc/lib/clspv/math/clc_sw_fma.cl index 306aad5..e73f53e 100644 --- a/clc/lib/clspv/math/clc_sw_fma.cl +++ b/clc/lib/clspv/math/clc_sw_fma.cl
@@ -17,6 +17,7 @@ #include "clc/integer/clc_hadd.h" #include "clc/integer/clc_mul_hi.h" #include "clc/integer/definitions.h" +#include "clc/math/clc_flush_if_daz.h" #include "clc/math/clc_mad.h" #include "clc/math/math.h" #include "clc/relational/clc_isinf.h" @@ -127,9 +128,9 @@ return c; } - a = __clc_flush_denormal_if_not_supported(a); - b = __clc_flush_denormal_if_not_supported(b); - c = __clc_flush_denormal_if_not_supported(c); + a = __clc_flush_if_daz(a); + b = __clc_flush_if_daz(b); + c = __clc_flush_if_daz(c); if (a == 0.0f || b == 0.0f) { return c;
diff --git a/clc/lib/generic/CMakeLists.txt b/clc/lib/generic/CMakeLists.txt index af6a556..9f93fdb 100644 --- a/clc/lib/generic/CMakeLists.txt +++ b/clc/lib/generic/CMakeLists.txt
@@ -66,6 +66,7 @@ math/clc_atan2pi.cl math/clc_atanh.cl math/clc_atanpi.cl + math/clc_canonicalize.cl math/clc_cbrt.cl math/clc_ceil.cl math/clc_copysign.cl @@ -87,6 +88,7 @@ math/clc_fabs.cl math/clc_fdim.cl math/clc_floor.cl + math/clc_flush_if_daz.cl math/clc_fma.cl math/clc_fmax.cl math/clc_fmin.cl
diff --git a/clc/lib/generic/math/clc_canonicalize.cl b/clc/lib/generic/math/clc_canonicalize.cl new file mode 100644 index 0000000..5b0ae52 --- /dev/null +++ b/clc/lib/generic/math/clc_canonicalize.cl
@@ -0,0 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clc/math/clc_canonicalize.h" + +#define __CLC_FUNCTION __clc_canonicalize +#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_canonicalize +#define __CLC_BODY "clc/shared/unary_def.inc" + +#include "clc/math/gentype.inc"
diff --git a/clc/lib/generic/math/clc_flush_if_daz.cl b/clc/lib/generic/math/clc_flush_if_daz.cl new file mode 100644 index 0000000..1abb183 --- /dev/null +++ b/clc/lib/generic/math/clc_flush_if_daz.cl
@@ -0,0 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "clc/clc_convert.h" +#include "clc/math/clc_canonicalize.h" +#include "clc/math/clc_flush_if_daz.h" +#include "clc/math/clc_subnormal_config.h" +#include "clc/math/math.h" + +#define __CLC_BODY "clc_flush_if_daz.inc" +#include "clc/math/gentype.inc"
diff --git a/clc/lib/generic/math/clc_flush_if_daz.inc b/clc/lib/generic/math/clc_flush_if_daz.inc new file mode 100644 index 0000000..5ae1b39 --- /dev/null +++ b/clc/lib/generic/math/clc_flush_if_daz.inc
@@ -0,0 +1,43 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if __CLC_FPSIZE == 16 +#define __CLC_GENTYPE_EXPBITS (__CLC_S_GENTYPE) EXPBITS_FP16 +#define __CLC_GENTYPE_MANTBITS (__CLC_S_GENTYPE) MANTBITS_FP16 +#define __CLC_GENTYPE_SIGNBIT (__CLC_S_GENTYPE) SIGNBIT_FP16 +#elif __CLC_FPSIZE == 32 +#define __CLC_GENTYPE_EXPBITS (__CLC_S_GENTYPE) EXPBITS_SP32 +#define __CLC_GENTYPE_MANTBITS (__CLC_S_GENTYPE) MANTBITS_SP32 +#define __CLC_GENTYPE_SIGNBIT (__CLC_S_GENTYPE) SIGNBIT_SP32 +#elif __CLC_FPSIZE == 64 +#define __CLC_GENTYPE_EXPBITS (__CLC_S_GENTYPE) EXPBITS_DP64 +#define __CLC_GENTYPE_MANTBITS (__CLC_S_GENTYPE) MANTBITS_DP64 +#define __CLC_GENTYPE_SIGNBIT (__CLC_S_GENTYPE) SIGNBIT_DP64 +#endif + +_CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_flush_if_daz(__CLC_GENTYPE x) { + if (!__CLC_GENTYPE_DENORMS_ARE_ZERO) + return x; + + // Hack around canonicalize not working on spirv. +#if defined(CLC_SPIRV) || defined(CLC_CLSPV) + __CLC_S_GENTYPE ix = __CLC_AS_S_GENTYPE(x); + __CLC_S_GENTYPE should_flush = + ((ix & __CLC_GENTYPE_EXPBITS) == (__CLC_S_GENTYPE)0) && + ((ix & __CLC_GENTYPE_MANTBITS) != (__CLC_S_GENTYPE)0); + __CLC_S_GENTYPE signbit = ix &= __CLC_GENTYPE_SIGNBIT; + __CLC_S_GENTYPE result = should_flush ? signbit : ix; + return __CLC_AS_GENTYPE(result); +#else + return __clc_canonicalize(x); +#endif +} + +#undef __CLC_GENTYPE_EXPBITS +#undef __CLC_GENTYPE_MANTBITS +#undef __CLC_GENTYPE_SIGNBIT
diff --git a/clc/lib/generic/math/clc_remquo.cl b/clc/lib/generic/math/clc_remquo.cl index 6f6a7fc..e254093 100644 --- a/clc/lib/generic/math/clc_remquo.cl +++ b/clc/lib/generic/math/clc_remquo.cl
@@ -10,6 +10,7 @@ #include "clc/integer/clc_clz.h" #include "clc/internal/clc.h" #include "clc/math/clc_floor.h" +#include "clc/math/clc_flush_if_daz.h" #include "clc/math/clc_fma.h" #include "clc/math/clc_ldexp.h" #include "clc/math/clc_subnormal_config.h"
diff --git a/clc/lib/generic/math/clc_remquo.inc b/clc/lib/generic/math/clc_remquo.inc index 3a76ffe..cf8a5eb 100644 --- a/clc/lib/generic/math/clc_remquo.inc +++ b/clc/lib/generic/math/clc_remquo.inc
@@ -8,8 +8,8 @@ _CLC_DEF _CLC_OVERLOAD float __clc_remquo(float x, float y, __CLC_ADDRESS_SPACE int *quo) { - x = __clc_flush_denormal_if_not_supported(x); - y = __clc_flush_denormal_if_not_supported(y); + x = __clc_flush_if_daz(x); + y = __clc_flush_if_daz(y); int ux = __clc_as_int(x); int ax = ux & EXSIGNBIT_SP32; float xa = __clc_as_float(ax);