Author: Fraser Cormack Date: 2025-03-04T14:16:16Z New Revision: e5d5503e4efa48b61194b1e70e469aba91297bec
URL: https://github.com/llvm/llvm-project/commit/e5d5503e4efa48b61194b1e70e469aba91297bec DIFF: https://github.com/llvm/llvm-project/commit/e5d5503e4efa48b61194b1e70e469aba91297bec.diff LOG: [libclc] Move hypot to CLC library; optimize (#129551) This was already nominally in the CLC library; this commit just formally moves it over. It simultaneously optimizes it for vector types by avoiding scalarization. Added: libclc/clc/include/clc/math/clc_hypot.h libclc/clc/lib/generic/math/clc_hypot.cl libclc/clc/lib/generic/math/clc_hypot.inc Modified: libclc/clc/lib/generic/SOURCES libclc/clspv/lib/SOURCES libclc/generic/lib/SOURCES libclc/generic/lib/math/hypot.cl libclc/spirv/lib/SOURCES Removed: libclc/generic/include/math/clc_hypot.h libclc/generic/lib/math/clc_hypot.cl ################################################################################ diff --git a/libclc/clc/include/clc/math/clc_hypot.h b/libclc/clc/include/clc/math/clc_hypot.h new file mode 100644 index 0000000000000..c09e89bb2f071 --- /dev/null +++ b/libclc/clc/include/clc/math/clc_hypot.h @@ -0,0 +1,12 @@ +#ifndef __CLC_MATH_CLC_HYPOT_H__ +#define __CLC_MATH_CLC_HYPOT_H__ + +#define __CLC_BODY <clc/shared/binary_decl.inc> +#define __CLC_FUNCTION __clc_hypot + +#include <clc/math/gentype.inc> + +#undef __CLC_BODY +#undef __CLC_FUNCTION + +#endif // __CLC_MATH_CLC_HYPOT_H__ diff --git a/libclc/clc/lib/generic/SOURCES b/libclc/clc/lib/generic/SOURCES index 206c7c18ce1a1..f7688d0442253 100644 --- a/libclc/clc/lib/generic/SOURCES +++ b/libclc/clc/lib/generic/SOURCES @@ -23,6 +23,7 @@ math/clc_fabs.cl math/clc_fma.cl math/clc_floor.cl math/clc_frexp.cl +math/clc_hypot.cl math/clc_ldexp.cl math/clc_log.cl math/clc_log10.cl diff --git a/libclc/clc/lib/generic/math/clc_hypot.cl b/libclc/clc/lib/generic/math/clc_hypot.cl new file mode 100644 index 0000000000000..3bbe5a984efe5 --- /dev/null +++ b/libclc/clc/lib/generic/math/clc_hypot.cl @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include <clc/clc_convert.h> +#include <clc/clcmacro.h> +#include <clc/integer/clc_abs.h> +#include <clc/internal/clc.h> +#include <clc/math/clc_fma.h> +#include <clc/math/clc_mad.h> +#include <clc/math/clc_sqrt.h> +#include <clc/math/clc_subnormal_config.h> +#include <clc/math/math.h> +#include <clc/relational/clc_isnan.h> +#include <clc/shared/clc_clamp.h> + +#define __CLC_BODY <clc_hypot.inc> +#include <clc/math/gentype.inc> +#undef __CLC_BODY diff --git a/libclc/clc/lib/generic/math/clc_hypot.inc b/libclc/clc/lib/generic/math/clc_hypot.inc new file mode 100644 index 0000000000000..72a989646ff38 --- /dev/null +++ b/libclc/clc/lib/generic/math/clc_hypot.inc @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +// Returns sqrt(x*x + y*y) with no overflow or underflow unless the result +// warrants it + +#if __CLC_FPSIZE == 32 +_CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_hypot(__CLC_GENTYPE x, + __CLC_GENTYPE y) { + __CLC_UINTN ux = __CLC_AS_UINTN(x); + __CLC_UINTN aux = ux & EXSIGNBIT_SP32; + __CLC_UINTN uy = __CLC_AS_UINTN(y); + __CLC_UINTN auy = uy & EXSIGNBIT_SP32; + __CLC_INTN c = aux > auy; + ux = c ? aux : auy; + uy = c ? auy : aux; + + __CLC_INTN xexp = __clc_clamp( + __CLC_AS_INTN(ux >> EXPSHIFTBITS_SP32) - EXPBIAS_SP32, -126, 126); + __CLC_GENTYPE fx_exp = + __CLC_AS_GENTYPE((xexp + EXPBIAS_SP32) << EXPSHIFTBITS_SP32); + __CLC_GENTYPE fi_exp = + __CLC_AS_GENTYPE((-xexp + EXPBIAS_SP32) << EXPSHIFTBITS_SP32); + __CLC_GENTYPE fx = __CLC_AS_GENTYPE(ux) * fi_exp; + __CLC_GENTYPE fy = __CLC_AS_GENTYPE(uy) * fi_exp; + + __CLC_GENTYPE retval = __clc_sqrt(__clc_mad(fx, fx, fy * fy)) * fx_exp; + + retval = (ux > PINFBITPATT_SP32 || uy == 0) ? __CLC_AS_GENTYPE(ux) : retval; + retval = (ux == PINFBITPATT_SP32 || uy == PINFBITPATT_SP32) + ? __CLC_AS_GENTYPE((__CLC_UINTN)PINFBITPATT_SP32) + : retval; + return retval; +} + +#elif __CLC_FPSIZE == 64 + +_CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_hypot(__CLC_GENTYPE x, + __CLC_GENTYPE y) { + __CLC_ULONGN ux = __CLC_AS_ULONGN(x) & ~SIGNBIT_DP64; + __CLC_INTN xexp = __CLC_CONVERT_INTN(ux >> EXPSHIFTBITS_DP64); + x = __CLC_AS_GENTYPE(ux); + + __CLC_ULONGN uy = __CLC_AS_ULONGN(y) & ~SIGNBIT_DP64; + __CLC_INTN yexp = __CLC_CONVERT_INTN(uy >> EXPSHIFTBITS_DP64); + y = __CLC_AS_GENTYPE(uy); + + __CLC_LONGN c = __CLC_CONVERT_LONGN(xexp > EXPBIAS_DP64 + 500 || + yexp > EXPBIAS_DP64 + 500); + __CLC_GENTYPE preadjust = c ? 0x1.0p-600 : 1.0; + __CLC_GENTYPE postadjust = c ? 0x1.0p+600 : 1.0; + + c = __CLC_CONVERT_LONGN(xexp < EXPBIAS_DP64 - 500 || + yexp < EXPBIAS_DP64 - 500); + preadjust = c ? 0x1.0p+600 : preadjust; + postadjust = c ? 0x1.0p-600 : postadjust; + + __CLC_GENTYPE ax = x * preadjust; + __CLC_GENTYPE ay = y * preadjust; + + // The post adjust may overflow, but this can't be avoided in any case + __CLC_GENTYPE r = __clc_sqrt(__clc_fma(ax, ax, ay * ay)) * postadjust; + + // If the diff erence in exponents between x and y is large + __CLC_GENTYPE s = x + y; + c = __CLC_CONVERT_LONGN(__clc_abs(xexp - yexp) > MANTLENGTH_DP64 + 1); + r = c ? s : r; + + // Check for NaN + c = __clc_isnan(x) || __clc_isnan(y); + r = c ? __CLC_AS_GENTYPE((__CLC_ULONGN)QNANBITPATT_DP64) : r; + + // If either is Inf, we must return Inf + c = x == __CLC_AS_GENTYPE((__CLC_ULONGN)PINFBITPATT_DP64) || + y == __CLC_AS_GENTYPE((__CLC_ULONGN)PINFBITPATT_DP64); + r = c ? __CLC_AS_GENTYPE((__CLC_ULONGN)PINFBITPATT_DP64) : r; + + return r; +} + +#elif __CLC_FPSIZE == 16 + +_CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_hypot(__CLC_GENTYPE x, + __CLC_GENTYPE y) { + return __CLC_CONVERT_GENTYPE( + __clc_hypot(__CLC_CONVERT_FLOATN(x), __CLC_CONVERT_FLOATN(y))); +} + +#endif diff --git a/libclc/clspv/lib/SOURCES b/libclc/clspv/lib/SOURCES index 15c437beb03b9..fa9e68f6985ea 100644 --- a/libclc/clspv/lib/SOURCES +++ b/libclc/clspv/lib/SOURCES @@ -18,7 +18,6 @@ subnormal_config.cl ../../generic/lib/math/cbrt.cl ../../generic/lib/math/clc_exp10.cl ../../generic/lib/math/clc_fmod.cl -../../generic/lib/math/clc_hypot.cl ../../generic/lib/math/clc_pow.cl ../../generic/lib/math/clc_pown.cl ../../generic/lib/math/clc_powr.cl diff --git a/libclc/generic/include/math/clc_hypot.h b/libclc/generic/include/math/clc_hypot.h deleted file mode 100644 index 66901e5d692b6..0000000000000 --- a/libclc/generic/include/math/clc_hypot.h +++ /dev/null @@ -1,5 +0,0 @@ -#define __CLC_FUNCTION __clc_hypot -#define __CLC_BODY <clc/math/binary_decl_tt.inc> -#include <clc/math/gentype.inc> -#undef __CLC_BODY -#undef __CLC_FUNCTION diff --git a/libclc/generic/lib/SOURCES b/libclc/generic/lib/SOURCES index 5b1c85d6ef75b..15aafee79dfec 100644 --- a/libclc/generic/lib/SOURCES +++ b/libclc/generic/lib/SOURCES @@ -127,7 +127,6 @@ math/half_rsqrt.cl math/half_sin.cl math/half_sqrt.cl math/half_tan.cl -math/clc_hypot.cl math/hypot.cl math/ilogb.cl math/ldexp.cl diff --git a/libclc/generic/lib/math/clc_hypot.cl b/libclc/generic/lib/math/clc_hypot.cl deleted file mode 100644 index fdf1e7ffa1def..0000000000000 --- a/libclc/generic/lib/math/clc_hypot.cl +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2014 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include <clc/clc.h> -#include <clc/clcmacro.h> -#include <clc/integer/clc_abs.h> -#include <clc/math/clc_fma.h> -#include <clc/math/clc_mad.h> -#include <clc/math/clc_subnormal_config.h> -#include <clc/math/math.h> -#include <clc/math/clc_sqrt.h> -#include <clc/relational/clc_isnan.h> -#include <clc/shared/clc_clamp.h> -#include <math/clc_hypot.h> - -// Returns sqrt(x*x + y*y) with no overflow or underflow unless the result -// warrants it -_CLC_DEF _CLC_OVERLOAD float __clc_hypot(float x, float y) { - uint ux = as_uint(x); - uint aux = ux & EXSIGNBIT_SP32; - uint uy = as_uint(y); - uint auy = uy & EXSIGNBIT_SP32; - float retval; - int c = aux > auy; - ux = c ? aux : auy; - uy = c ? auy : aux; - - int xexp = - __clc_clamp((int)(ux >> EXPSHIFTBITS_SP32) - EXPBIAS_SP32, -126, 126); - float fx_exp = as_float((xexp + EXPBIAS_SP32) << EXPSHIFTBITS_SP32); - float fi_exp = as_float((-xexp + EXPBIAS_SP32) << EXPSHIFTBITS_SP32); - float fx = as_float(ux) * fi_exp; - float fy = as_float(uy) * fi_exp; - retval = __clc_sqrt(__clc_mad(fx, fx, fy * fy)) * fx_exp; - - retval = ux > PINFBITPATT_SP32 | uy == 0 ? as_float(ux) : retval; - retval = ux == PINFBITPATT_SP32 | uy == PINFBITPATT_SP32 - ? as_float(PINFBITPATT_SP32) - : retval; - return retval; -} -_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_hypot, float, float) - -#ifdef cl_khr_fp64 -_CLC_DEF _CLC_OVERLOAD double __clc_hypot(double x, double y) { - ulong ux = as_ulong(x) & ~SIGNBIT_DP64; - int xexp = ux >> EXPSHIFTBITS_DP64; - x = as_double(ux); - - ulong uy = as_ulong(y) & ~SIGNBIT_DP64; - int yexp = uy >> EXPSHIFTBITS_DP64; - y = as_double(uy); - - int c = xexp > EXPBIAS_DP64 + 500 | yexp > EXPBIAS_DP64 + 500; - double preadjust = c ? 0x1.0p-600 : 1.0; - double postadjust = c ? 0x1.0p+600 : 1.0; - - c = xexp < EXPBIAS_DP64 - 500 | yexp < EXPBIAS_DP64 - 500; - preadjust = c ? 0x1.0p+600 : preadjust; - postadjust = c ? 0x1.0p-600 : postadjust; - - double ax = x * preadjust; - double ay = y * preadjust; - - // The post adjust may overflow, but this can't be avoided in any case - double r = __clc_sqrt(__clc_fma(ax, ax, ay * ay)) * postadjust; - - // If the diff erence in exponents between x and y is large - double s = x + y; - c = __clc_abs(xexp - yexp) > MANTLENGTH_DP64 + 1; - r = c ? s : r; - - // Check for NaN - // c = x != x | y != y; - c = __clc_isnan(x) | __clc_isnan(y); - r = c ? as_double(QNANBITPATT_DP64) : r; - - // If either is Inf, we must return Inf - c = x == as_double(PINFBITPATT_DP64) | y == as_double(PINFBITPATT_DP64); - r = c ? as_double(PINFBITPATT_DP64) : r; - - return r; -} - -_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_hypot, double, - double) -#endif diff --git a/libclc/generic/lib/math/hypot.cl b/libclc/generic/lib/math/hypot.cl index 8339ec75c002b..b2563a18a3585 100644 --- a/libclc/generic/lib/math/hypot.cl +++ b/libclc/generic/lib/math/hypot.cl @@ -1,7 +1,6 @@ #include <clc/clc.h> +#include <clc/math/clc_hypot.h> -#include <math/clc_hypot.h> - -#define __CLC_FUNC hypot -#define __CLC_BODY <clc_sw_binary.inc> +#define FUNCTION hypot +#define __CLC_BODY <clc/shared/binary_def.inc> #include <clc/math/gentype.inc> diff --git a/libclc/spirv/lib/SOURCES b/libclc/spirv/lib/SOURCES index 35eef0f364707..8378587d52bf3 100644 --- a/libclc/spirv/lib/SOURCES +++ b/libclc/spirv/lib/SOURCES @@ -48,7 +48,6 @@ math/fma.cl ../../generic/lib/math/frexp.cl ../../generic/lib/math/half_rsqrt.cl ../../generic/lib/math/half_sqrt.cl -../../generic/lib/math/clc_hypot.cl ../../generic/lib/math/hypot.cl ../../generic/lib/math/ilogb.cl ../../generic/lib/math/ldexp.cl _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
