Author: Fraser Cormack Date: 2025-04-01T09:19:50+01:00 New Revision: 7a2b160e76e23d8fa62750af20e1e25f08803784
URL: https://github.com/llvm/llvm-project/commit/7a2b160e76e23d8fa62750af20e1e25f08803784 DIFF: https://github.com/llvm/llvm-project/commit/7a2b160e76e23d8fa62750af20e1e25f08803784.diff LOG: [libclc] Move rootn to the CLC library; optimize (#133735) The function was already nominally in the CLC namespace; this commit just moves it over. This commit also vectorizes the builtin to avoid scalarization. Added: libclc/clc/include/clc/math/clc_rootn.h libclc/clc/lib/generic/math/clc_rootn.cl libclc/clc/lib/generic/math/clc_rootn.inc Modified: libclc/clc/lib/generic/SOURCES libclc/clspv/lib/SOURCES libclc/generic/include/clc/math/rootn.h libclc/generic/lib/SOURCES libclc/generic/lib/math/rootn.cl libclc/spirv/lib/SOURCES Removed: libclc/generic/include/clc/math/rootn.inc libclc/generic/include/math/clc_rootn.h libclc/generic/include/math/clc_rootn.inc libclc/generic/lib/math/clc_rootn.cl libclc/generic/lib/math/rootn.inc ################################################################################ diff --git a/libclc/generic/include/math/clc_rootn.h b/libclc/clc/include/clc/math/clc_rootn.h similarity index 64% rename from libclc/generic/include/math/clc_rootn.h rename to libclc/clc/include/clc/math/clc_rootn.h index e4c9bd847302b..b1e69025d97a2 100644 --- a/libclc/generic/include/math/clc_rootn.h +++ b/libclc/clc/include/clc/math/clc_rootn.h @@ -6,6 +6,15 @@ // //===----------------------------------------------------------------------===// -#define __CLC_BODY <math/clc_rootn.inc> +#ifndef __CLC_MATH_CLC_ROOTN_H__ +#define __CLC_MATH_CLC_ROOTN_H__ + +#define __CLC_BODY <clc/math/binary_decl_with_int_second_arg.inc> +#define __CLC_FUNCTION __clc_rootn + #include <clc/math/gentype.inc> + #undef __CLC_BODY +#undef __CLC_FUNCTION + +#endif // __CLC_MATH_CLC_ROOTN_H__ diff --git a/libclc/clc/lib/generic/SOURCES b/libclc/clc/lib/generic/SOURCES index 3c648edd52a7c..6a1ba95362220 100644 --- a/libclc/clc/lib/generic/SOURCES +++ b/libclc/clc/lib/generic/SOURCES @@ -52,6 +52,7 @@ math/clc_powr.cl math/clc_remainder.cl math/clc_remquo.cl math/clc_rint.cl +math/clc_rootn.cl math/clc_round.cl math/clc_rsqrt.cl math/clc_sincos_helpers.cl diff --git a/libclc/clc/lib/generic/math/clc_rootn.cl b/libclc/clc/lib/generic/math/clc_rootn.cl new file mode 100644 index 0000000000000..da397cf66da62 --- /dev/null +++ b/libclc/clc/lib/generic/math/clc_rootn.cl @@ -0,0 +1,21 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/clc_convert.h> +#include <clc/float/definitions.h> +#include <clc/internal/clc.h> +#include <clc/math/clc_fabs.h> +#include <clc/math/clc_fma.h> +#include <clc/math/clc_ldexp.h> +#include <clc/math/clc_mad.h> +#include <clc/math/clc_subnormal_config.h> +#include <clc/math/math.h> +#include <clc/math/tables.h> + +#define __CLC_BODY <clc_rootn.inc> +#include <clc/math/gentype.inc> diff --git a/libclc/clc/lib/generic/math/clc_rootn.inc b/libclc/clc/lib/generic/math/clc_rootn.inc new file mode 100644 index 0000000000000..4fa56bdd84abe --- /dev/null +++ b/libclc/clc/lib/generic/math/clc_rootn.inc @@ -0,0 +1,403 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Computes pow using log and exp +// +// x^y = exp(y * log(x)) +// +// We take care not to lose precision in the intermediate steps. +// +// When computing log, calculate it in splits: +// +// r = f * (p_invead + p_inv_tail) +// r = rh + rt +// +// Calculate log polynomial using r, in end addition, do: +// +// poly = poly + ((rh-r) + rt) +// +// lth = -r +// ltt = ((xexp * log2_t) - poly) + logT +// lt = lth + ltt +// +// lh = (xexp * log2_h) + logH +// l = lh + lt +// +// Calculate final log answer as gh and gt: +// +// gh = l & higher-half bits +// gt = (((ltt - (lt - lth)) + ((lh - l) + lt)) + (l - gh)) +// +// yh = y & higher-half bits +// yt = y - yh +// +// Before entering computation of exp: +// +// vs = ((yt*gt + yt*gh) + yh*gt) +// v = vs + yh*gh +// vt = ((yh*gh - v) + vs) +// +// In calculation of exp, add vt to r that is used for poly. +// +// At the end of exp, do: +// +// ((((expT * poly) + expT) + expH*poly) + expH) +// +//===----------------------------------------------------------------------===// + +#if __CLC_FPSIZE == 32 + +_CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x, + __CLC_INTN ny) { + __CLC_GENTYPE y = MATH_RECIP(__CLC_CONVERT_GENTYPE(ny)); + + __CLC_INTN ix = __CLC_AS_INTN(x); + __CLC_INTN ax = ix & EXSIGNBIT_SP32; + __CLC_INTN xpos = ix == ax; + + __CLC_INTN iy = __CLC_AS_INTN(y); + __CLC_INTN ay = iy & EXSIGNBIT_SP32; + __CLC_INTN ypos = iy == ay; + + // Extra precise log calculation + // First handle case that x is close to 1 + __CLC_GENTYPE r = 1.0f - __CLC_AS_GENTYPE(ax); + __CLC_INTN near1 = __clc_fabs(r) < 0x1.0p-4f; + __CLC_GENTYPE r2 = r * r; + + // Coefficients are just 1/3, 1/4, 1/5 and 1/6 + __CLC_GENTYPE poly = __clc_mad( + r, + __clc_mad(r, + __clc_mad(r, __clc_mad(r, 0x1.24924ap-3f, 0x1.555556p-3f), + 0x1.99999ap-3f), + 0x1.000000p-2f), + 0x1.555556p-2f); + + poly *= r2 * r; + + __CLC_GENTYPE lth_near1 = -r2 * 0.5f; + __CLC_GENTYPE ltt_near1 = -poly; + __CLC_GENTYPE lt_near1 = lth_near1 + ltt_near1; + __CLC_GENTYPE lh_near1 = -r; + __CLC_GENTYPE l_near1 = lh_near1 + lt_near1; + + // Computations for x not near 1 + __CLC_INTN m = __CLC_CONVERT_INTN(ax >> EXPSHIFTBITS_SP32) - EXPBIAS_SP32; + __CLC_GENTYPE mf = __CLC_CONVERT_GENTYPE(m); + __CLC_INTN ixs = __CLC_AS_INTN(__CLC_AS_GENTYPE(ax | 0x3f800000) - 1.0f); + __CLC_GENTYPE mfs = __CLC_CONVERT_GENTYPE((ixs >> EXPSHIFTBITS_SP32) - 253); + __CLC_INTN c = m == -127; + __CLC_INTN ixn = c ? ixs : ax; + __CLC_GENTYPE mfn = c ? mfs : mf; + + __CLC_INTN indx = (ixn & 0x007f0000) + ((ixn & 0x00008000) << 1); + + // F - Y + __CLC_GENTYPE f = __CLC_AS_GENTYPE(0x3f000000 | indx) - + __CLC_AS_GENTYPE(0x3f000000 | (ixn & MANTBITS_SP32)); + + indx = indx >> 16; + __CLC_GENTYPE rh = f * USE_TABLE(log_inv_tbl_ep_head, indx); + __CLC_GENTYPE rt = f * USE_TABLE(log_inv_tbl_ep_tail, indx); + ; + r = rh + rt; + + poly = __clc_mad(r, __clc_mad(r, 0x1.0p-2f, 0x1.555556p-2f), 0x1.0p-1f) * + (r * r); + poly += (rh - r) + rt; + + const __CLC_GENTYPE LOG2_HEAD = 0x1.62e000p-1f; // 0.693115234 + const __CLC_GENTYPE LOG2_TAIL = 0x1.0bfbe8p-15f; // 0.0000319461833 + __CLC_GENTYPE lth = -r; + __CLC_GENTYPE ltt = + __clc_mad(mfn, LOG2_TAIL, -poly) + USE_TABLE(loge_tbl_hi, indx); + __CLC_GENTYPE lt = lth + ltt; + __CLC_GENTYPE lh = __clc_mad(mfn, LOG2_HEAD, USE_TABLE(loge_tbl_lo, indx)); + __CLC_GENTYPE l = lh + lt; + + // Select near 1 or not + lth = near1 ? lth_near1 : lth; + ltt = near1 ? ltt_near1 : ltt; + lt = near1 ? lt_near1 : lt; + lh = near1 ? lh_near1 : lh; + l = near1 ? l_near1 : l; + + __CLC_GENTYPE gh = __CLC_AS_GENTYPE(__CLC_AS_UINTN(l) & 0xfffff000); + __CLC_GENTYPE gt = ((ltt - (lt - lth)) + ((lh - l) + lt)) + (l - gh); + + __CLC_GENTYPE yh = __CLC_AS_GENTYPE(__CLC_AS_UINTN(iy) & 0xfffff000); + + __CLC_GENTYPE fny = __CLC_CONVERT_GENTYPE(ny); + __CLC_GENTYPE fnyh = __CLC_AS_GENTYPE(__CLC_AS_UINTN(fny) & 0xfffff000); + __CLC_GENTYPE fnyt = __CLC_CONVERT_GENTYPE(ny - __CLC_CONVERT_INTN(fnyh)); + __CLC_GENTYPE yt = + MATH_DIVIDE(__clc_mad(-fnyt, yh, __clc_mad(-fnyh, yh, 1.0f)), fny); + + __CLC_GENTYPE ylogx_s = __clc_mad(gt, yh, __clc_mad(gh, yt, yt * gt)); + __CLC_GENTYPE ylogx = __clc_mad(yh, gh, ylogx_s); + __CLC_GENTYPE ylogx_t = __clc_mad(yh, gh, -ylogx) + ylogx_s; + + // Extra precise exp of ylogx + const __CLC_GENTYPE R_64_BY_LOG2 = + 0x1.715476p+6f; // 64/log2 : 92.332482616893657 + __CLC_INTN n = __CLC_CONVERT_INTN(ylogx * R_64_BY_LOG2); + __CLC_GENTYPE nf = __CLC_CONVERT_GENTYPE(n); + + __CLC_INTN j = n & 0x3f; + m = n >> 6; + __CLC_INTN m2 = m << EXPSHIFTBITS_SP32; + + // log2/64 lead: 0.0108032227 + const __CLC_GENTYPE R_LOG2_BY_64_LD = 0x1.620000p-7f; + // log2/64 tail: 0.0000272020388 + const __CLC_GENTYPE R_LOG2_BY_64_TL = 0x1.c85fdep-16f; + r = __clc_mad(nf, -R_LOG2_BY_64_TL, __clc_mad(nf, -R_LOG2_BY_64_LD, ylogx)) + + ylogx_t; + + // Truncated Taylor series for e^r + poly = __clc_mad(__clc_mad(__clc_mad(r, 0x1.555556p-5f, 0x1.555556p-3f), r, + 0x1.000000p-1f), + r * r, r); + + __CLC_GENTYPE exph = USE_TABLE(exp_tbl_ep_head, j); + __CLC_GENTYPE expt = USE_TABLE(exp_tbl_ep_tail, j); + + __CLC_GENTYPE expylogx = + __clc_mad(exph, poly, __clc_mad(expt, poly, expt)) + exph; + __CLC_GENTYPE sexpylogx = + __clc_fp32_subnormals_supported() + ? expylogx * __CLC_AS_GENTYPE((__CLC_INTN)0x1 << (m + 149)) + : 0.0f; + + __CLC_GENTYPE texpylogx = __CLC_AS_GENTYPE(__CLC_AS_INTN(expylogx) + m2); + expylogx = m < -125 ? sexpylogx : texpylogx; + + // Result is +-Inf if (ylogx + ylogx_t) > 128*log2 + expylogx = ((ylogx > 0x1.62e430p+6f) | + (ylogx == 0x1.62e430p+6f & ylogx_t > -0x1.05c610p-22f)) + ? __CLC_AS_GENTYPE((__CLC_UINTN)PINFBITPATT_SP32) + : expylogx; + + // Result is 0 if ylogx < -149*log2 + expylogx = ylogx < -0x1.9d1da0p+6f ? 0.0f : expylogx; + + // Classify y: + // inty = 0 means not an integer. + // inty = 1 means odd integer. + // inty = 2 means even integer. + + __CLC_INTN inty = 2 - (ny & 1); + + __CLC_GENTYPE signval = + __CLC_AS_GENTYPE((__CLC_AS_UINTN(expylogx) ^ SIGNBIT_SP32)); + expylogx = ((inty == 1) & !xpos) ? signval : expylogx; + __CLC_INTN ret = __CLC_AS_INTN(expylogx); + + // Corner case handling + __CLC_BIT_INTN x_is_ninf = ix == (__CLC_INTN)NINFBITPATT_SP32; + __CLC_BIT_INTN x_is_pinf = ix == (__CLC_INTN)PINFBITPATT_SP32; + + ret = (!xpos & (inty == 2)) ? __CLC_AS_INTN((__CLC_GENTYPE)__CLC_GENTYPE_NAN) + : ret; + __CLC_INTN xinf = + xpos ? (__CLC_INTN)PINFBITPATT_SP32 : (__CLC_INTN)NINFBITPATT_SP32; + ret = ((ax == 0) & !ypos & (inty == 1)) ? xinf : ret; + ret = ((ax == 0) & !ypos & (inty == 2)) ? PINFBITPATT_SP32 : ret; + ret = ((ax == 0) & ypos & (inty == 2)) ? 0 : ret; + __CLC_INTN xzero = xpos ? 0 : (__CLC_INTN)0x80000000; + ret = ((ax == 0) & ypos & (inty == 1)) ? xzero : ret; + ret = (x_is_ninf & ypos & (inty == 1)) ? (__CLC_INTN)NINFBITPATT_SP32 : ret; + ret = (x_is_ninf & !ypos & (inty == 1)) ? (__CLC_INTN)0x80000000 : ret; + ret = (x_is_pinf & !ypos) ? 0 : ret; + ret = (x_is_pinf & ypos) ? PINFBITPATT_SP32 : ret; + ret = ax > PINFBITPATT_SP32 ? ix : ret; + ret = ny == 0 ? __CLC_AS_INTN((__CLC_GENTYPE)__CLC_GENTYPE_NAN) : ret; + + return __CLC_AS_GENTYPE(ret); +} + +#elif __CLC_FPSIZE == 64 + +_CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x, + __CLC_INTN ny) { + const __CLC_GENTYPE real_log2_tail = 5.76999904754328540596e-08; + const __CLC_GENTYPE real_log2_lead = 6.93147122859954833984e-01; + + __CLC_GENTYPE dny = __CLC_CONVERT_GENTYPE(ny); + __CLC_GENTYPE y = 1.0 / dny; + + __CLC_LONGN ux = __CLC_AS_LONGN(x); + __CLC_LONGN ax = __CLC_AS_LONGN(__clc_fabs(x)); + __CLC_BIT_INTN xpos = ax == ux; + + __CLC_LONGN uy = __CLC_AS_LONGN(y); + __CLC_LONGN ay = __CLC_AS_LONGN(__clc_fabs(y)); + __CLC_BIT_INTN ypos = ay == uy; + + // Extended precision log + __CLC_GENTYPE v, vt; + { + __CLC_INTN exp = __CLC_CONVERT_INTN(ax >> 52) - 1023; + __CLC_INTN mask_exp_1023 = exp == -1023; + __CLC_GENTYPE xexp = __CLC_CONVERT_GENTYPE(exp); + __CLC_LONGN mantissa = ax & 0x000FFFFFFFFFFFFFL; + + __CLC_LONGN temp_ux = + __CLC_AS_LONGN(__CLC_AS_GENTYPE(0x3ff0000000000000L | mantissa) - 1.0); + exp = __CLC_CONVERT_INTN((temp_ux & 0x7FF0000000000000L) >> 52) - 2045; + __CLC_GENTYPE xexp1 = __CLC_CONVERT_GENTYPE(exp); + __CLC_LONGN mantissa1 = temp_ux & 0x000FFFFFFFFFFFFFL; + + xexp = __CLC_CONVERT_LONGN(mask_exp_1023) ? xexp1 : xexp; + mantissa = __CLC_CONVERT_LONGN(mask_exp_1023) ? mantissa1 : mantissa; + + __CLC_LONGN rax = (mantissa & 0x000ff00000000000) + + ((mantissa & 0x0000080000000000) << 1); + __CLC_INTN index = __CLC_CONVERT_INTN(rax >> 44); + + __CLC_GENTYPE F = __CLC_AS_GENTYPE(rax | 0x3FE0000000000000L); + __CLC_GENTYPE Y = __CLC_AS_GENTYPE(mantissa | 0x3FE0000000000000L); + __CLC_GENTYPE f = F - Y; + __CLC_GENTYPE log_h = USE_TABLE(log_f_inv_tbl_head, index); + __CLC_GENTYPE log_t = USE_TABLE(log_f_inv_tbl_tail, index); + __CLC_GENTYPE f_inv = (log_h + log_t) * f; + __CLC_GENTYPE r1 = + __CLC_AS_GENTYPE(__CLC_AS_ULONGN(f_inv) & 0xfffffffff8000000L); + __CLC_GENTYPE r2 = __clc_fma(-F, r1, f) * (log_h + log_t); + __CLC_GENTYPE r = r1 + r2; + + __CLC_GENTYPE poly = __clc_fma( + r, + __clc_fma(r, + __clc_fma(r, __clc_fma(r, 1.0 / 7.0, 1.0 / 6.0), 1.0 / 5.0), + 1.0 / 4.0), + 1.0 / 3.0); + poly = poly * r * r * r; + + __CLC_GENTYPE hr1r1 = 0.5 * r1 * r1; + __CLC_GENTYPE poly0h = r1 + hr1r1; + __CLC_GENTYPE poly0t = r1 - poly0h + hr1r1; + poly = __clc_fma(r1, r2, __clc_fma(0.5 * r2, r2, poly)) + r2 + poly0t; + + log_h = USE_TABLE(powlog_tbl_head, index); + log_t = USE_TABLE(powlog_tbl_tail, index); + + __CLC_GENTYPE resT_t = __clc_fma(xexp, real_log2_tail, +log_t) - poly; + __CLC_GENTYPE resT = resT_t - poly0h; + __CLC_GENTYPE resH = __clc_fma(xexp, real_log2_lead, log_h); + __CLC_GENTYPE resT_h = poly0h; + + __CLC_GENTYPE H = resT + resH; + __CLC_GENTYPE H_h = + __CLC_AS_GENTYPE(__CLC_AS_ULONGN(H) & 0xfffffffff8000000L); + __CLC_GENTYPE T = + (resH - H + resT) + (resT_t - (resT + resT_h)) + (H - H_h); + H = H_h; + + __CLC_GENTYPE y_head = + __CLC_AS_GENTYPE(__CLC_AS_ULONGN(uy) & 0xfffffffff8000000L); + __CLC_GENTYPE y_tail = y - y_head; + + __CLC_GENTYPE fnyh = + __CLC_AS_GENTYPE(__CLC_AS_ULONGN(dny) & 0xfffffffffff00000); + __CLC_GENTYPE fnyt = __CLC_CONVERT_GENTYPE(ny - __CLC_CONVERT_INTN(fnyh)); + y_tail = __clc_fma(-fnyt, y_head, __clc_fma(-fnyh, y_head, 1.0)) / dny; + + __CLC_GENTYPE temp = __clc_fma(y_tail, H, __clc_fma(y_head, T, y_tail * T)); + v = __clc_fma(y_head, H, temp); + vt = __clc_fma(y_head, H, -v) + temp; + } + + // Now calculate exp of (v,vt) + + __CLC_GENTYPE expv; + { + const __CLC_GENTYPE max_exp_arg = 709.782712893384; + const __CLC_GENTYPE min_exp_arg = -745.1332191019411; + const __CLC_GENTYPE sixtyfour_by_lnof2 = 92.33248261689366; + const __CLC_GENTYPE lnof2_by_64_head = 0.010830424260348081; + const __CLC_GENTYPE lnof2_by_64_tail = -4.359010638708991e-10; + + __CLC_GENTYPE temp = v * sixtyfour_by_lnof2; + __CLC_INTN n = __CLC_CONVERT_INTN(temp); + __CLC_GENTYPE dn = __CLC_CONVERT_GENTYPE(n); + __CLC_INTN j = n & 0x0000003f; + __CLC_INTN m = n >> 6; + + __CLC_GENTYPE f1 = USE_TABLE(two_to_jby64_ep_tbl_head, j); + __CLC_GENTYPE f2 = USE_TABLE(two_to_jby64_ep_tbl_tail, j); + __CLC_GENTYPE f = f1 + f2; + + __CLC_GENTYPE r1 = __clc_fma(dn, -lnof2_by_64_head, v); + __CLC_GENTYPE r2 = dn * lnof2_by_64_tail; + __CLC_GENTYPE r = (r1 + r2) + vt; + + __CLC_GENTYPE q = + __clc_fma(r, + __clc_fma(r, + __clc_fma(r, + __clc_fma(r, 1.38889490863777199667e-03, + 8.33336798434219616221e-03), + 4.16666666662260795726e-02), + 1.66666666665260878863e-01), + 5.00000000000000008883e-01); + q = __clc_fma(r * r, q, r); + + expv = __clc_fma(f, q, f2) + f1; + expv = __clc_ldexp(expv, m); + + expv = v > max_exp_arg ? __CLC_AS_GENTYPE((__CLC_ULONGN)0x7FF0000000000000L) + : expv; + expv = v < min_exp_arg ? 0.0 : expv; + } + + // See whether y is an integer. + // inty = 0 means not an integer. + // inty = 1 means odd integer. + // inty = 2 means even integer. + + __CLC_LONGN inty = __CLC_CONVERT_LONGN(2 - (ny & 1)); + + expv *= ((inty == 1) & !xpos) ? -1.0 : 1.0; + + __CLC_LONGN ret = __CLC_AS_LONGN(expv); + + // Now all the edge cases + __CLC_BIT_INTN x_is_ninf = ux == (__CLC_LONGN)NINFBITPATT_DP64; + __CLC_BIT_INTN x_is_pinf = ux == (__CLC_LONGN)PINFBITPATT_DP64; + ret = (!xpos & (inty == 2)) ? __CLC_AS_LONGN((__CLC_GENTYPE)__CLC_GENTYPE_NAN) + : ret; + __CLC_LONGN xinf = + xpos ? (__CLC_LONGN)PINFBITPATT_DP64 : (__CLC_LONGN)NINFBITPATT_DP64; + ret = ((ax == 0L) & !ypos & (inty == 1)) ? xinf : ret; + ret = + ((ax == 0L) & !ypos & (inty == 2)) ? (__CLC_LONGN)PINFBITPATT_DP64 : ret; + ret = ((ax == 0L) & ypos & (inty == 2)) ? 0L : ret; + __CLC_LONGN xzero = xpos ? 0L : (__CLC_LONGN)0x8000000000000000L; + ret = ((ax == 0L) & ypos & (inty == 1)) ? xzero : ret; + ret = (x_is_ninf & ypos & (inty == 1)) ? (__CLC_LONGN)NINFBITPATT_DP64 : ret; + ret = (x_is_ninf & !ypos & (inty == 1)) ? (__CLC_LONGN)0x8000000000000000L + : ret; + ret = (x_is_pinf & !ypos) ? 0L : ret; + ret = (x_is_pinf & ypos) ? (__CLC_LONGN)PINFBITPATT_DP64 : ret; + ret = ax > (__CLC_LONGN)PINFBITPATT_DP64 ? ux : ret; + ret = __CLC_CONVERT_LONGN(ny == 0) + ? __CLC_AS_LONGN((__CLC_GENTYPE)__CLC_GENTYPE_NAN) + : ret; + return __CLC_AS_GENTYPE(ret); +} + +#elif __CLC_FPSIZE == 16 + +_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x, + __CLC_INTN y) { + return __CLC_CONVERT_GENTYPE(__clc_rootn(__CLC_CONVERT_FLOATN(x), y)); +} + +#endif diff --git a/libclc/clspv/lib/SOURCES b/libclc/clspv/lib/SOURCES index c0a84f2a554f0..1c86fb0fbc8e8 100644 --- a/libclc/clspv/lib/SOURCES +++ b/libclc/clspv/lib/SOURCES @@ -17,7 +17,6 @@ subnormal_config.cl ../../generic/lib/math/atanpi.cl ../../generic/lib/math/cbrt.cl ../../generic/lib/math/clc_exp10.cl -../../generic/lib/math/clc_rootn.cl ../../generic/lib/math/clc_tan.cl ../../generic/lib/math/clc_tanpi.cl ../../generic/lib/math/cos.cl diff --git a/libclc/generic/include/clc/math/rootn.h b/libclc/generic/include/clc/math/rootn.h index fecbec1d2d8fb..41df8cd9a8210 100644 --- a/libclc/generic/include/clc/math/rootn.h +++ b/libclc/generic/include/clc/math/rootn.h @@ -6,6 +6,10 @@ // //===----------------------------------------------------------------------===// -#define __CLC_BODY <clc/math/rootn.inc> +#define __CLC_BODY <clc/math/binary_decl_with_int_second_arg.inc> +#define __CLC_FUNCTION rootn + #include <clc/math/gentype.inc> + #undef __CLC_BODY +#undef __CLC_FUNCTION diff --git a/libclc/generic/include/clc/math/rootn.inc b/libclc/generic/include/clc/math/rootn.inc deleted file mode 100644 index 667b904a14b75..0000000000000 --- a/libclc/generic/include/clc/math/rootn.inc +++ /dev/null @@ -1,9 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE rootn(__CLC_GENTYPE a, __CLC_INTN b); diff --git a/libclc/generic/include/math/clc_rootn.inc b/libclc/generic/include/math/clc_rootn.inc deleted file mode 100644 index 391f9069c7e46..0000000000000 --- a/libclc/generic/include/math/clc_rootn.inc +++ /dev/null @@ -1,9 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE a, __CLC_INTN b); diff --git a/libclc/generic/lib/SOURCES b/libclc/generic/lib/SOURCES index 1c3aff16035c5..a93444af0c954 100644 --- a/libclc/generic/lib/SOURCES +++ b/libclc/generic/lib/SOURCES @@ -162,7 +162,6 @@ math/powr.cl math/remainder.cl math/remquo.cl math/rint.cl -math/clc_rootn.cl math/rootn.cl math/round.cl math/rsqrt.cl diff --git a/libclc/generic/lib/math/clc_rootn.cl b/libclc/generic/lib/math/clc_rootn.cl deleted file mode 100644 index f642792e1275f..0000000000000 --- a/libclc/generic/lib/math/clc_rootn.cl +++ /dev/null @@ -1,389 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include <clc/clc.h> -#include <clc/clc_convert.h> -#include <clc/clcmacro.h> -#include <clc/math/clc_fabs.h> -#include <clc/math/clc_fma.h> -#include <clc/math/clc_mad.h> -#include <clc/math/clc_subnormal_config.h> -#include <clc/math/math.h> -#include <clc/math/tables.h> - -// compute pow using log and exp -// x^y = exp(y * log(x)) -// -// we take care not to lose precision in the intermediate steps -// -// When computing log, calculate it in splits, -// -// r = f * (p_invead + p_inv_tail) -// r = rh + rt -// -// calculate log polynomial using r, in end addition, do -// poly = poly + ((rh-r) + rt) -// -// lth = -r -// ltt = ((xexp * log2_t) - poly) + logT -// lt = lth + ltt -// -// lh = (xexp * log2_h) + logH -// l = lh + lt -// -// Calculate final log answer as gh and gt, -// gh = l & higher-half bits -// gt = (((ltt - (lt - lth)) + ((lh - l) + lt)) + (l - gh)) -// -// yh = y & higher-half bits -// yt = y - yh -// -// Before entering computation of exp, -// vs = ((yt*gt + yt*gh) + yh*gt) -// v = vs + yh*gh -// vt = ((yh*gh - v) + vs) -// -// In calculation of exp, add vt to r that is used for poly -// At the end of exp, do -// ((((expT * poly) + expT) + expH*poly) + expH) - -_CLC_DEF _CLC_OVERLOAD float __clc_rootn(float x, int ny) { - float y = MATH_RECIP((float)ny); - - int ix = __clc_as_int(x); - int ax = ix & EXSIGNBIT_SP32; - int xpos = ix == ax; - - int iy = __clc_as_int(y); - int ay = iy & EXSIGNBIT_SP32; - int ypos = iy == ay; - - // Extra precise log calculation - // First handle case that x is close to 1 - float r = 1.0f - __clc_as_float(ax); - int near1 = __clc_fabs(r) < 0x1.0p-4f; - float r2 = r * r; - - // Coefficients are just 1/3, 1/4, 1/5 and 1/6 - float poly = __clc_mad( - r, - __clc_mad(r, - __clc_mad(r, __clc_mad(r, 0x1.24924ap-3f, 0x1.555556p-3f), - 0x1.99999ap-3f), - 0x1.000000p-2f), - 0x1.555556p-2f); - - poly *= r2 * r; - - float lth_near1 = -r2 * 0.5f; - float ltt_near1 = -poly; - float lt_near1 = lth_near1 + ltt_near1; - float lh_near1 = -r; - float l_near1 = lh_near1 + lt_near1; - - // Computations for x not near 1 - int m = (int)(ax >> EXPSHIFTBITS_SP32) - EXPBIAS_SP32; - float mf = (float)m; - int ixs = __clc_as_int(__clc_as_float(ax | 0x3f800000) - 1.0f); - float mfs = (float)((ixs >> EXPSHIFTBITS_SP32) - 253); - int c = m == -127; - int ixn = c ? ixs : ax; - float mfn = c ? mfs : mf; - - int indx = (ixn & 0x007f0000) + ((ixn & 0x00008000) << 1); - - // F - Y - float f = __clc_as_float(0x3f000000 | indx) - - __clc_as_float(0x3f000000 | (ixn & MANTBITS_SP32)); - - indx = indx >> 16; - float2 tv; - tv.s0 = USE_TABLE(log_inv_tbl_ep_head, indx); - tv.s1 = USE_TABLE(log_inv_tbl_ep_tail, indx); - float rh = f * tv.s0; - float rt = f * tv.s1; - r = rh + rt; - - poly = __clc_mad(r, __clc_mad(r, 0x1.0p-2f, 0x1.555556p-2f), 0x1.0p-1f) * - (r * r); - poly += (rh - r) + rt; - - const float LOG2_HEAD = 0x1.62e000p-1f; // 0.693115234 - const float LOG2_TAIL = 0x1.0bfbe8p-15f; // 0.0000319461833 - tv.s0 = USE_TABLE(loge_tbl_lo, indx); - tv.s1 = USE_TABLE(loge_tbl_hi, indx); - float lth = -r; - float ltt = __clc_mad(mfn, LOG2_TAIL, -poly) + tv.s1; - float lt = lth + ltt; - float lh = __clc_mad(mfn, LOG2_HEAD, tv.s0); - float l = lh + lt; - - // Select near 1 or not - lth = near1 ? lth_near1 : lth; - ltt = near1 ? ltt_near1 : ltt; - lt = near1 ? lt_near1 : lt; - lh = near1 ? lh_near1 : lh; - l = near1 ? l_near1 : l; - - float gh = __clc_as_float(__clc_as_int(l) & 0xfffff000); - float gt = ((ltt - (lt - lth)) + ((lh - l) + lt)) + (l - gh); - - float yh = __clc_as_float(iy & 0xfffff000); - - float fny = (float)ny; - float fnyh = __clc_as_float(__clc_as_int(fny) & 0xfffff000); - float fnyt = (float)(ny - (int)fnyh); - float yt = MATH_DIVIDE(__clc_mad(-fnyt, yh, __clc_mad(-fnyh, yh, 1.0f)), fny); - - float ylogx_s = __clc_mad(gt, yh, __clc_mad(gh, yt, yt * gt)); - float ylogx = __clc_mad(yh, gh, ylogx_s); - float ylogx_t = __clc_mad(yh, gh, -ylogx) + ylogx_s; - - // Extra precise exp of ylogx - const float R_64_BY_LOG2 = 0x1.715476p+6f; // 64/log2 : 92.332482616893657 - int n = __clc_convert_int(ylogx * R_64_BY_LOG2); - float nf = (float)n; - - int j = n & 0x3f; - m = n >> 6; - int m2 = m << EXPSHIFTBITS_SP32; - - // log2/64 lead: 0.0108032227 - const float R_LOG2_BY_64_LD = 0x1.620000p-7f; - // log2/64 tail: 0.0000272020388 - const float R_LOG2_BY_64_TL = 0x1.c85fdep-16f; - r = __clc_mad(nf, -R_LOG2_BY_64_TL, __clc_mad(nf, -R_LOG2_BY_64_LD, ylogx)) + - ylogx_t; - - // Truncated Taylor series for e^r - poly = __clc_mad(__clc_mad(__clc_mad(r, 0x1.555556p-5f, 0x1.555556p-3f), r, - 0x1.000000p-1f), - r * r, r); - - tv.s0 = USE_TABLE(exp_tbl_ep_head, j); - tv.s1 = USE_TABLE(exp_tbl_ep_tail, j); - - float expylogx = - __clc_mad(tv.s0, poly, __clc_mad(tv.s1, poly, tv.s1)) + tv.s0; - float sexpylogx = __clc_fp32_subnormals_supported() - ? expylogx * __clc_as_float(0x1 << (m + 149)) - : 0.0f; - - float texpylogx = __clc_as_float(__clc_as_int(expylogx) + m2); - expylogx = m < -125 ? sexpylogx : texpylogx; - - // Result is +-Inf if (ylogx + ylogx_t) > 128*log2 - expylogx = ((ylogx > 0x1.62e430p+6f) | - (ylogx == 0x1.62e430p+6f & ylogx_t > -0x1.05c610p-22f)) - ? __clc_as_float(PINFBITPATT_SP32) - : expylogx; - - // Result is 0 if ylogx < -149*log2 - expylogx = ylogx < -0x1.9d1da0p+6f ? 0.0f : expylogx; - - // Classify y: - // inty = 0 means not an integer. - // inty = 1 means odd integer. - // inty = 2 means even integer. - - int inty = 2 - (ny & 1); - - float signval = __clc_as_float((__clc_as_uint(expylogx) ^ SIGNBIT_SP32)); - expylogx = ((inty == 1) & !xpos) ? signval : expylogx; - int ret = __clc_as_int(expylogx); - - // Corner case handling - ret = (!xpos & (inty == 2)) ? QNANBITPATT_SP32 : ret; - int xinf = xpos ? PINFBITPATT_SP32 : NINFBITPATT_SP32; - ret = ((ax == 0) & !ypos & (inty == 1)) ? xinf : ret; - ret = ((ax == 0) & !ypos & (inty == 2)) ? PINFBITPATT_SP32 : ret; - ret = ((ax == 0) & ypos & (inty == 2)) ? 0 : ret; - int xzero = xpos ? 0 : 0x80000000; - ret = ((ax == 0) & ypos & (inty == 1)) ? xzero : ret; - ret = - ((ix == NINFBITPATT_SP32) & ypos & (inty == 1)) ? NINFBITPATT_SP32 : ret; - ret = ((ix == NINFBITPATT_SP32) & !ypos & (inty == 1)) ? 0x80000000 : ret; - ret = ((ix == PINFBITPATT_SP32) & !ypos) ? 0 : ret; - ret = ((ix == PINFBITPATT_SP32) & ypos) ? PINFBITPATT_SP32 : ret; - ret = ax > PINFBITPATT_SP32 ? ix : ret; - ret = ny == 0 ? QNANBITPATT_SP32 : ret; - - return __clc_as_float(ret); -} -_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_rootn, float, int) - -#ifdef cl_khr_fp64 -_CLC_DEF _CLC_OVERLOAD double __clc_rootn(double x, int ny) { - const double real_log2_tail = 5.76999904754328540596e-08; - const double real_log2_lead = 6.93147122859954833984e-01; - - double dny = (double)ny; - double y = 1.0 / dny; - - long ux = __clc_as_long(x); - long ax = ux & (~SIGNBIT_DP64); - int xpos = ax == ux; - - long uy = __clc_as_long(y); - long ay = uy & (~SIGNBIT_DP64); - int ypos = ay == uy; - - // Extended precision log - double v, vt; - { - int exp = (int)(ax >> 52) - 1023; - int mask_exp_1023 = exp == -1023; - double xexp = (double)exp; - long mantissa = ax & 0x000FFFFFFFFFFFFFL; - - long temp_ux = - __clc_as_long(__clc_as_double(0x3ff0000000000000L | mantissa) - 1.0); - exp = ((temp_ux & 0x7FF0000000000000L) >> 52) - 2045; - double xexp1 = (double)exp; - long mantissa1 = temp_ux & 0x000FFFFFFFFFFFFFL; - - xexp = mask_exp_1023 ? xexp1 : xexp; - mantissa = mask_exp_1023 ? mantissa1 : mantissa; - - long rax = (mantissa & 0x000ff00000000000) + - ((mantissa & 0x0000080000000000) << 1); - int index = rax >> 44; - - double F = __clc_as_double(rax | 0x3FE0000000000000L); - double Y = __clc_as_double(mantissa | 0x3FE0000000000000L); - double f = F - Y; - double log_h = USE_TABLE(log_f_inv_tbl_head, index); - double log_t = USE_TABLE(log_f_inv_tbl_tail, index); - double f_inv = (log_h + log_t) * f; - double r1 = __clc_as_double(__clc_as_long(f_inv) & 0xfffffffff8000000L); - double r2 = __clc_fma(-F, r1, f) * (log_h + log_t); - double r = r1 + r2; - - double poly = __clc_fma( - r, - __clc_fma(r, - __clc_fma(r, __clc_fma(r, 1.0 / 7.0, 1.0 / 6.0), 1.0 / 5.0), - 1.0 / 4.0), - 1.0 / 3.0); - poly = poly * r * r * r; - - double hr1r1 = 0.5 * r1 * r1; - double poly0h = r1 + hr1r1; - double poly0t = r1 - poly0h + hr1r1; - poly = __clc_fma(r1, r2, __clc_fma(0.5 * r2, r2, poly)) + r2 + poly0t; - - log_h = USE_TABLE(powlog_tbl_head, index); - log_t = USE_TABLE(powlog_tbl_tail, index); - - double resT_t = __clc_fma(xexp, real_log2_tail, +log_t) - poly; - double resT = resT_t - poly0h; - double resH = __clc_fma(xexp, real_log2_lead, log_h); - double resT_h = poly0h; - - double H = resT + resH; - double H_h = __clc_as_double(__clc_as_long(H) & 0xfffffffff8000000L); - double T = (resH - H + resT) + (resT_t - (resT + resT_h)) + (H - H_h); - H = H_h; - - double y_head = __clc_as_double(uy & 0xfffffffff8000000L); - double y_tail = y - y_head; - - double fnyh = __clc_as_double(__clc_as_long(dny) & 0xfffffffffff00000); - double fnyt = (double)(ny - (int)fnyh); - y_tail = __clc_fma(-fnyt, y_head, __clc_fma(-fnyh, y_head, 1.0)) / dny; - - double temp = __clc_fma(y_tail, H, __clc_fma(y_head, T, y_tail * T)); - v = __clc_fma(y_head, H, temp); - vt = __clc_fma(y_head, H, -v) + temp; - } - - // Now calculate exp of (v,vt) - - double expv; - { - const double max_exp_arg = 709.782712893384; - const double min_exp_arg = -745.1332191019411; - const double sixtyfour_by_lnof2 = 92.33248261689366; - const double lnof2_by_64_head = 0.010830424260348081; - const double lnof2_by_64_tail = -4.359010638708991e-10; - - double temp = v * sixtyfour_by_lnof2; - int n = (int)temp; - double dn = (double)n; - int j = n & 0x0000003f; - int m = n >> 6; - - double f1 = USE_TABLE(two_to_jby64_ep_tbl_head, j); - double f2 = USE_TABLE(two_to_jby64_ep_tbl_tail, j); - double f = f1 + f2; - - double r1 = __clc_fma(dn, -lnof2_by_64_head, v); - double r2 = dn * lnof2_by_64_tail; - double r = (r1 + r2) + vt; - - double q = - __clc_fma(r, - __clc_fma(r, - __clc_fma(r, - __clc_fma(r, 1.38889490863777199667e-03, - 8.33336798434219616221e-03), - 4.16666666662260795726e-02), - 1.66666666665260878863e-01), - 5.00000000000000008883e-01); - q = __clc_fma(r * r, q, r); - - expv = __clc_fma(f, q, f2) + f1; - expv = ldexp(expv, m); - - expv = v > max_exp_arg ? __clc_as_double(0x7FF0000000000000L) : expv; - expv = v < min_exp_arg ? 0.0 : expv; - } - - // See whether y is an integer. - // inty = 0 means not an integer. - // inty = 1 means odd integer. - // inty = 2 means even integer. - - int inty = 2 - (ny & 1); - - expv *= ((inty == 1) & !xpos) ? -1.0 : 1.0; - - long ret = __clc_as_long(expv); - - // Now all the edge cases - ret = (!xpos & (inty == 2)) ? QNANBITPATT_DP64 : ret; - long xinf = xpos ? PINFBITPATT_DP64 : NINFBITPATT_DP64; - ret = ((ax == 0L) & !ypos & (inty == 1)) ? xinf : ret; - ret = ((ax == 0L) & !ypos & (inty == 2)) ? PINFBITPATT_DP64 : ret; - ret = ((ax == 0L) & ypos & (inty == 2)) ? 0L : ret; - long xzero = xpos ? 0L : 0x8000000000000000L; - ret = ((ax == 0L) & ypos & (inty == 1)) ? xzero : ret; - ret = - ((ux == NINFBITPATT_DP64) & ypos & (inty == 1)) ? NINFBITPATT_DP64 : ret; - ret = ((ux == NINFBITPATT_DP64) & !ypos & (inty == 1)) ? 0x8000000000000000L - : ret; - ret = ((ux == PINFBITPATT_DP64) & !ypos) ? 0L : ret; - ret = ((ux == PINFBITPATT_DP64) & ypos) ? PINFBITPATT_DP64 : ret; - ret = ax > PINFBITPATT_DP64 ? ux : ret; - ret = ny == 0 ? QNANBITPATT_DP64 : ret; - return __clc_as_double(ret); -} -_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_rootn, double, int) -#endif - -#ifdef cl_khr_fp16 - -#pragma OPENCL EXTENSION cl_khr_fp16 : enable - -_CLC_OVERLOAD _CLC_DEF half __clc_rootn(half x, int y) { - return (half)__clc_rootn((float)x, y); -} - -_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_rootn, half, int); - -#endif diff --git a/libclc/generic/lib/math/rootn.cl b/libclc/generic/lib/math/rootn.cl index 92c3de46843c5..b14248636e149 100644 --- a/libclc/generic/lib/math/rootn.cl +++ b/libclc/generic/lib/math/rootn.cl @@ -7,8 +7,8 @@ //===----------------------------------------------------------------------===// #include <clc/clc.h> +#include <clc/math/clc_rootn.h> -#include <math/clc_rootn.h> - -#define __CLC_BODY <rootn.inc> +#define FUNCTION rootn +#define __CLC_BODY <clc/math/binary_def_with_int_second_arg.inc> #include <clc/math/gentype.inc> diff --git a/libclc/generic/lib/math/rootn.inc b/libclc/generic/lib/math/rootn.inc deleted file mode 100644 index a01e72017ac22..0000000000000 --- a/libclc/generic/lib/math/rootn.inc +++ /dev/null @@ -1,11 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE rootn(__CLC_GENTYPE x, __CLC_INTN y) { - return __clc_rootn(x, y); -} diff --git a/libclc/spirv/lib/SOURCES b/libclc/spirv/lib/SOURCES index 921a257814817..ad9f44a6149ae 100644 --- a/libclc/spirv/lib/SOURCES +++ b/libclc/spirv/lib/SOURCES @@ -63,7 +63,6 @@ math/fma.cl ../../generic/lib/math/powr.cl ../../generic/lib/math/remainder.cl ../../generic/lib/math/remquo.cl -../../generic/lib/math/clc_rootn.cl ../../generic/lib/math/rootn.cl ../../generic/lib/math/sin.cl ../../generic/lib/math/sincos.cl _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
