2017-09-06 16:42:00 +00:00
|
|
|
/* Configuration for math routines.
|
2023-01-06 21:08:04 +00:00
|
|
|
Copyright (C) 2017-2023 Free Software Foundation, Inc.
|
2017-09-06 16:42:00 +00:00
|
|
|
This file is part of the GNU C Library.
|
|
|
|
|
|
|
|
The GNU C Library is free software; you can redistribute it and/or
|
|
|
|
modify it under the terms of the GNU Lesser General Public
|
|
|
|
License as published by the Free Software Foundation; either
|
|
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
|
|
|
|
The GNU C Library is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
Lesser General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
|
|
License along with the GNU C Library; if not, see
|
Prefer https to http for gnu.org and fsf.org URLs
Also, change sources.redhat.com to sourceware.org.
This patch was automatically generated by running the following shell
script, which uses GNU sed, and which avoids modifying files imported
from upstream:
sed -ri '
s,(http|ftp)(://(.*\.)?(gnu|fsf|sourceware)\.org($|[^.]|\.[^a-z])),https\2,g
s,(http|ftp)(://(.*\.)?)sources\.redhat\.com($|[^.]|\.[^a-z]),https\2sourceware.org\4,g
' \
$(find $(git ls-files) -prune -type f \
! -name '*.po' \
! -name 'ChangeLog*' \
! -path COPYING ! -path COPYING.LIB \
! -path manual/fdl-1.3.texi ! -path manual/lgpl-2.1.texi \
! -path manual/texinfo.tex ! -path scripts/config.guess \
! -path scripts/config.sub ! -path scripts/install-sh \
! -path scripts/mkinstalldirs ! -path scripts/move-if-change \
! -path INSTALL ! -path locale/programs/charmap-kw.h \
! -path po/libc.pot ! -path sysdeps/gnu/errlist.c \
! '(' -name configure \
-execdir test -f configure.ac -o -f configure.in ';' ')' \
! '(' -name preconfigure \
-execdir test -f preconfigure.ac ';' ')' \
-print)
and then by running 'make dist-prepare' to regenerate files built
from the altered files, and then executing the following to cleanup:
chmod a+x sysdeps/unix/sysv/linux/riscv/configure
# Omit irrelevant whitespace and comment-only changes,
# perhaps from a slightly-different Autoconf version.
git checkout -f \
sysdeps/csky/configure \
sysdeps/hppa/configure \
sysdeps/riscv/configure \
sysdeps/unix/sysv/linux/csky/configure
# Omit changes that caused a pre-commit check to fail like this:
# remote: *** error: sysdeps/powerpc/powerpc64/ppc-mcount.S: trailing lines
git checkout -f \
sysdeps/powerpc/powerpc64/ppc-mcount.S \
sysdeps/unix/sysv/linux/s390/s390-64/syscall.S
# Omit change that caused a pre-commit check to fail like this:
# remote: *** error: sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S: last line does not end in newline
git checkout -f sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S
2019-09-07 05:40:42 +00:00
|
|
|
<https://www.gnu.org/licenses/>. */
|
2017-09-06 16:42:00 +00:00
|
|
|
|
|
|
|
#ifndef _MATH_CONFIG_H
|
|
|
|
#define _MATH_CONFIG_H
|
|
|
|
|
|
|
|
#include <math.h>
|
|
|
|
#include <math_private.h>
|
2017-09-04 16:55:33 +00:00
|
|
|
#include <nan-high-order-bit.h>
|
2017-09-06 16:42:00 +00:00
|
|
|
#include <stdint.h>
|
|
|
|
|
|
|
|
#ifndef WANT_ROUNDING
|
|
|
|
/* Correct special case results in non-nearest rounding modes. */
|
|
|
|
# define WANT_ROUNDING 1
|
|
|
|
#endif
|
|
|
|
#ifndef WANT_ERRNO
|
|
|
|
/* Set errno according to ISO C with (math_errhandling & MATH_ERRNO) != 0. */
|
|
|
|
# define WANT_ERRNO 1
|
|
|
|
#endif
|
|
|
|
#ifndef WANT_ERRNO_UFLOW
|
|
|
|
/* Set errno to ERANGE if result underflows to 0 (in all rounding modes). */
|
|
|
|
# define WANT_ERRNO_UFLOW (WANT_ROUNDING && WANT_ERRNO)
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifndef TOINT_INTRINSICS
|
2018-07-04 11:29:29 +00:00
|
|
|
/* When set, the roundtoint and converttoint functions are provided with
|
|
|
|
the semantics documented below. */
|
2017-09-06 16:42:00 +00:00
|
|
|
# define TOINT_INTRINSICS 0
|
|
|
|
#endif
|
2018-07-04 11:29:29 +00:00
|
|
|
|
|
|
|
#if TOINT_INTRINSICS
|
|
|
|
/* Round x to nearest int in all rounding modes, ties have to be rounded
|
|
|
|
consistently with converttoint so the results match. If the result
|
|
|
|
would be outside of [-2^31, 2^31-1] then the semantics is unspecified. */
|
|
|
|
static inline double_t
|
|
|
|
roundtoint (double_t x);
|
|
|
|
|
|
|
|
/* Convert x to nearest int in all rounding modes, ties have to be rounded
|
|
|
|
consistently with roundtoint. If the result is not representible in an
|
|
|
|
int32_t then the semantics is unspecified. */
|
|
|
|
static inline int32_t
|
|
|
|
converttoint (double_t x);
|
2017-09-06 16:42:00 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
static inline uint32_t
|
|
|
|
asuint (float f)
|
|
|
|
{
|
|
|
|
union
|
|
|
|
{
|
|
|
|
float f;
|
|
|
|
uint32_t i;
|
|
|
|
} u = {f};
|
|
|
|
return u.i;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline float
|
|
|
|
asfloat (uint32_t i)
|
|
|
|
{
|
|
|
|
union
|
|
|
|
{
|
|
|
|
uint32_t i;
|
|
|
|
float f;
|
|
|
|
} u = {i};
|
|
|
|
return u.f;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline uint64_t
|
|
|
|
asuint64 (double f)
|
|
|
|
{
|
|
|
|
union
|
|
|
|
{
|
|
|
|
double f;
|
|
|
|
uint64_t i;
|
|
|
|
} u = {f};
|
|
|
|
return u.i;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline double
|
|
|
|
asdouble (uint64_t i)
|
|
|
|
{
|
|
|
|
union
|
|
|
|
{
|
|
|
|
uint64_t i;
|
|
|
|
double f;
|
|
|
|
} u = {i};
|
|
|
|
return u.f;
|
|
|
|
}
|
|
|
|
|
2021-04-05 17:49:47 +00:00
|
|
|
static inline int
|
|
|
|
issignalingf_inline (float x)
|
|
|
|
{
|
|
|
|
uint32_t ix = asuint (x);
|
|
|
|
if (HIGH_ORDER_BIT_IS_SET_FOR_SNAN)
|
|
|
|
return (ix & 0x7fc00000) == 0x7fc00000;
|
|
|
|
return 2 * (ix ^ 0x00400000) > 2 * 0x7fc00000UL;
|
|
|
|
}
|
|
|
|
|
math: Improve fmodf
This uses a new algorithm similar to already proposed earlier [1].
With x = mx * 2^ex and y = my * 2^ey (mx, my, ex, ey being integers),
the simplest implementation is:
mx * 2^ex == 2 * mx * 2^(ex - 1)
while (ex > ey)
{
mx *= 2;
--ex;
mx %= my;
}
With mx/my being mantissa of double floating pointer, on each step the
argument reduction can be improved 8 (which is sizeof of uint32_t minus
MANTISSA_WIDTH plus the signal bit):
while (ex > ey)
{
mx << 8;
ex -= 8;
mx %= my;
} */
The implementation uses builtin clz and ctz, along with shifts to
convert hx/hy back to doubles. Different than the original patch,
this path assume modulo/divide operation is slow, so use multiplication
with invert values.
I see the following performance improvements using fmod benchtests
(result only show the 'mean' result):
Architecture | Input | master | patch
-----------------|-----------------|----------|--------
x86_64 (Ryzen 9) | subnormals | 17.2549 | 12.0318
x86_64 (Ryzen 9) | normal | 85.4096 | 49.9641
x86_64 (Ryzen 9) | close-exponents | 19.1072 | 15.8224
aarch64 (N1) | subnormal | 10.2182 | 6.81778
aarch64 (N1) | normal | 60.0616 | 20.3667
aarch64 (N1) | close-exponents | 11.5256 | 8.39685
I also see similar improvements on arm-linux-gnueabihf when running on
the N1 aarch64 chips, where it a lot of soft-fp implementation (for
modulo, and multiplication):
Architecture | Input | master | patch
-----------------|-----------------|----------|--------
armhf (N1) | subnormal | 11.6662 | 10.8955
armhf (N1) | normal | 69.2759 | 34.1524
armhf (N1) | close-exponents | 13.6472 | 18.2131
Instead of using the math_private.h definitions, I used the
math_config.h instead which is used on newer math implementations.
Co-authored-by: kirill <kirill.okhotnikov@gmail.com>
[1] https://sourceware.org/pipermail/libc-alpha/2020-November/119794.html
Reviewed-by: Wilco Dijkstra <Wilco.Dijkstra@arm.com>
2023-03-20 16:01:17 +00:00
|
|
|
#define BIT_WIDTH 32
|
|
|
|
#define MANTISSA_WIDTH 23
|
|
|
|
#define EXPONENT_WIDTH 8
|
|
|
|
#define MANTISSA_MASK 0x007fffff
|
|
|
|
#define EXPONENT_MASK 0x7f800000
|
|
|
|
#define EXP_MANT_MASK 0x7fffffff
|
|
|
|
#define QUIET_NAN_MASK 0x00400000
|
|
|
|
#define SIGN_MASK 0x80000000
|
|
|
|
|
|
|
|
static inline bool
|
|
|
|
is_nan (uint32_t x)
|
|
|
|
{
|
|
|
|
return (x & EXP_MANT_MASK) > EXPONENT_MASK;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline uint32_t
|
|
|
|
get_mantissa (uint32_t x)
|
|
|
|
{
|
|
|
|
return x & MANTISSA_MASK;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Convert integer number X, unbiased exponent EP, and sign S to double:
|
|
|
|
|
|
|
|
result = X * 2^(EP+1 - exponent_bias)
|
|
|
|
|
|
|
|
NB: zero is not supported. */
|
|
|
|
static inline double
|
|
|
|
make_float (uint32_t x, int ep, uint32_t s)
|
|
|
|
{
|
|
|
|
int lz = __builtin_clz (x) - EXPONENT_WIDTH;
|
|
|
|
x <<= lz;
|
|
|
|
ep -= lz;
|
|
|
|
|
|
|
|
if (__glibc_unlikely (ep < 0 || x == 0))
|
|
|
|
{
|
|
|
|
x >>= -ep;
|
|
|
|
ep = 0;
|
|
|
|
}
|
|
|
|
return asfloat (s + x + (ep << MANTISSA_WIDTH));
|
|
|
|
}
|
|
|
|
|
2018-06-13 14:07:21 +00:00
|
|
|
attribute_hidden float __math_oflowf (uint32_t);
|
|
|
|
attribute_hidden float __math_uflowf (uint32_t);
|
|
|
|
attribute_hidden float __math_may_uflowf (uint32_t);
|
|
|
|
attribute_hidden float __math_divzerof (uint32_t);
|
2017-09-06 16:42:00 +00:00
|
|
|
attribute_hidden float __math_invalidf (float);
|
2023-03-20 16:01:18 +00:00
|
|
|
attribute_hidden float __math_edomf (float x);
|
2017-09-06 16:42:00 +00:00
|
|
|
|
math: Optimized generic exp10f with wrappers
It is inspired by expf and reuses its tables and internal functions.
The error checks are inlined and errno setting is in separate tail
called functions, but the wrappers are kept in this patch to handle
the _LIB_VERSION==_SVID_ case.
Double precision arithmetics is used which is expected to be faster on
most targets (including soft-float) than using single precision and it
is easier to get good precision result with it.
Result for x86_64 (i7-4790K CPU @ 4.00GHz) are:
Before new code:
"exp10f": {
"workload-spec2017.wrf (adapted)": {
"duration": 4.0414e+09,
"iterations": 1.00128e+08,
"reciprocal-throughput": 26.6818,
"latency": 54.043,
"max-throughput": 3.74787e+07,
"min-throughput": 1.85038e+07
}
With new code:
"exp10f": {
"workload-spec2017.wrf (adapted)": {
"duration": 4.11951e+09,
"iterations": 1.23968e+08,
"reciprocal-throughput": 21.0581,
"latency": 45.4028,
"max-throughput": 4.74876e+07,
"min-throughput": 2.20251e+07
}
Result for aarch64 (A72 @ 2GHz) are:
Before new code:
"exp10f": {
"workload-spec2017.wrf (adapted)": {
"duration": 4.62362e+09,
"iterations": 3.3376e+07,
"reciprocal-throughput": 127.698,
"latency": 149.365,
"max-throughput": 7.831e+06,
"min-throughput": 6.69501e+06
}
With new code:
"exp10f": {
"workload-spec2017.wrf (adapted)": {
"duration": 4.29108e+09,
"iterations": 6.6752e+07,
"reciprocal-throughput": 51.2111,
"latency": 77.3568,
"max-throughput": 1.9527e+07,
"min-throughput": 1.29271e+07
}
Checked on x86_64-linux-gnu, powerpc64le-linux-gnu, aarch64-linux-gnu,
and sparc64-linux-gnu.
2020-04-08 20:32:28 +00:00
|
|
|
/* Shared between expf, exp2f, exp10f, and powf. */
|
2017-09-06 16:42:00 +00:00
|
|
|
#define EXP2F_TABLE_BITS 5
|
|
|
|
#define EXP2F_POLY_ORDER 3
|
|
|
|
extern const struct exp2f_data
|
|
|
|
{
|
|
|
|
uint64_t tab[1 << EXP2F_TABLE_BITS];
|
|
|
|
double shift_scaled;
|
|
|
|
double poly[EXP2F_POLY_ORDER];
|
|
|
|
double shift;
|
|
|
|
double invln2_scaled;
|
|
|
|
double poly_scaled[EXP2F_POLY_ORDER];
|
|
|
|
} __exp2f_data attribute_hidden;
|
|
|
|
|
2017-09-04 16:48:46 +00:00
|
|
|
#define LOGF_TABLE_BITS 4
|
|
|
|
#define LOGF_POLY_ORDER 4
|
|
|
|
extern const struct logf_data
|
|
|
|
{
|
|
|
|
struct
|
|
|
|
{
|
|
|
|
double invc, logc;
|
|
|
|
} tab[1 << LOGF_TABLE_BITS];
|
|
|
|
double ln2;
|
|
|
|
double poly[LOGF_POLY_ORDER - 1]; /* First order coefficient is 1. */
|
|
|
|
} __logf_data attribute_hidden;
|
|
|
|
|
2017-09-04 16:53:47 +00:00
|
|
|
#define LOG2F_TABLE_BITS 4
|
|
|
|
#define LOG2F_POLY_ORDER 4
|
|
|
|
extern const struct log2f_data
|
|
|
|
{
|
|
|
|
struct
|
|
|
|
{
|
|
|
|
double invc, logc;
|
|
|
|
} tab[1 << LOG2F_TABLE_BITS];
|
|
|
|
double poly[LOG2F_POLY_ORDER];
|
|
|
|
} __log2f_data attribute_hidden;
|
|
|
|
|
2017-09-04 16:55:33 +00:00
|
|
|
#define POWF_LOG2_TABLE_BITS 4
|
|
|
|
#define POWF_LOG2_POLY_ORDER 5
|
|
|
|
#if TOINT_INTRINSICS
|
|
|
|
# define POWF_SCALE_BITS EXP2F_TABLE_BITS
|
|
|
|
#else
|
|
|
|
# define POWF_SCALE_BITS 0
|
|
|
|
#endif
|
|
|
|
#define POWF_SCALE ((double) (1 << POWF_SCALE_BITS))
|
|
|
|
extern const struct powf_log2_data
|
|
|
|
{
|
|
|
|
struct
|
|
|
|
{
|
|
|
|
double invc, logc;
|
|
|
|
} tab[1 << POWF_LOG2_TABLE_BITS];
|
|
|
|
double poly[POWF_LOG2_POLY_ORDER];
|
|
|
|
} __powf_log2_data attribute_hidden;
|
|
|
|
|
2017-09-06 16:42:00 +00:00
|
|
|
#endif
|