1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2005-2011 David Schultz <das@FreeBSD.ORG> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <fenv.h> 30 #include <float.h> 31 #include <math.h> 32 33 #include "math_private.h" 34 35 #ifdef USE_BUILTIN_FMA 36 double 37 fma(double x, double y, double z) 38 { 39 return (__builtin_fma(x, y, z)); 40 } 41 #else 42 /* 43 * A struct dd represents a floating-point number with twice the precision 44 * of a double. We maintain the invariant that "hi" stores the 53 high-order 45 * bits of the result. 46 */ 47 struct dd { 48 double hi; 49 double lo; 50 }; 51 52 /* 53 * Compute a+b exactly, returning the exact result in a struct dd. We assume 54 * that both a and b are finite, but make no assumptions about their relative 55 * magnitudes. 56 */ 57 static inline struct dd 58 dd_add(double a, double b) 59 { 60 struct dd ret; 61 double s; 62 63 ret.hi = a + b; 64 s = ret.hi - a; 65 ret.lo = (a - (ret.hi - s)) + (b - s); 66 return (ret); 67 } 68 69 /* 70 * Compute a+b, with a small tweak: The least significant bit of the 71 * result is adjusted into a sticky bit summarizing all the bits that 72 * were lost to rounding. This adjustment negates the effects of double 73 * rounding when the result is added to another number with a higher 74 * exponent. For an explanation of round and sticky bits, see any reference 75 * on FPU design, e.g., 76 * 77 * J. Coonen. An Implementation Guide to a Proposed Standard for 78 * Floating-Point Arithmetic. Computer, vol. 13, no. 1, Jan 1980. 79 */ 80 static inline double 81 add_adjusted(double a, double b) 82 { 83 struct dd sum; 84 uint64_t hibits, lobits; 85 86 sum = dd_add(a, b); 87 if (sum.lo != 0) { 88 EXTRACT_WORD64(hibits, sum.hi); 89 if ((hibits & 1) == 0) { 90 /* hibits += (int)copysign(1.0, sum.hi * sum.lo) */ 91 EXTRACT_WORD64(lobits, sum.lo); 92 hibits += 1 - ((hibits ^ lobits) >> 62); 93 INSERT_WORD64(sum.hi, hibits); 94 } 95 } 96 return (sum.hi); 97 } 98 99 /* 100 * Compute ldexp(a+b, scale) with a single rounding error. It is assumed 101 * that the result will be subnormal, and care is taken to ensure that 102 * double rounding does not occur. 103 */ 104 static inline double 105 add_and_denormalize(double a, double b, int scale) 106 { 107 struct dd sum; 108 uint64_t hibits, lobits; 109 int bits_lost; 110 111 sum = dd_add(a, b); 112 113 /* 114 * If we are losing at least two bits of accuracy to denormalization, 115 * then the first lost bit becomes a round bit, and we adjust the 116 * lowest bit of sum.hi to make it a sticky bit summarizing all the 117 * bits in sum.lo. With the sticky bit adjusted, the hardware will 118 * break any ties in the correct direction. 119 * 120 * If we are losing only one bit to denormalization, however, we must 121 * break the ties manually. 122 */ 123 if (sum.lo != 0) { 124 EXTRACT_WORD64(hibits, sum.hi); 125 bits_lost = -((int)(hibits >> 52) & 0x7ff) - scale + 1; 126 if ((bits_lost != 1) ^ (int)(hibits & 1)) { 127 /* hibits += (int)copysign(1.0, sum.hi * sum.lo) */ 128 EXTRACT_WORD64(lobits, sum.lo); 129 hibits += 1 - (((hibits ^ lobits) >> 62) & 2); 130 INSERT_WORD64(sum.hi, hibits); 131 } 132 } 133 return (ldexp(sum.hi, scale)); 134 } 135 136 /* 137 * Compute a*b exactly, returning the exact result in a struct dd. We assume 138 * that both a and b are normalized, so no underflow or overflow will occur. 139 * The current rounding mode must be round-to-nearest. 140 */ 141 static inline struct dd 142 dd_mul(double a, double b) 143 { 144 static const double split = 0x1p27 + 1.0; 145 struct dd ret; 146 double ha, hb, la, lb, p, q; 147 148 p = a * split; 149 ha = a - p; 150 ha += p; 151 la = a - ha; 152 153 p = b * split; 154 hb = b - p; 155 hb += p; 156 lb = b - hb; 157 158 p = ha * hb; 159 q = ha * lb + la * hb; 160 161 ret.hi = p + q; 162 ret.lo = p - ret.hi + q + la * lb; 163 return (ret); 164 } 165 166 #ifdef _RENAME_FMA 167 double fma_sw(double, double, double); 168 #endif 169 170 /* 171 * Fused multiply-add: Compute x * y + z with a single rounding error. 172 * 173 * We use scaling to avoid overflow/underflow, along with the 174 * canonical precision-doubling technique adapted from: 175 * 176 * Dekker, T. A Floating-Point Technique for Extending the 177 * Available Precision. Numer. Math. 18, 224-242 (1971). 178 * 179 * This algorithm is sensitive to the rounding precision. FPUs such 180 * as the i387 must be set in double-precision mode if variables are 181 * to be stored in FP registers in order to avoid incorrect results. 182 * This is the default on FreeBSD, but not on many other systems. 183 * 184 * Hardware instructions should be used on architectures that support it, 185 * since this implementation will likely be several times slower. 186 */ 187 double 188 #ifdef _RENAME_FMA 189 fma_sw 190 #else 191 fma 192 #endif 193 (double x, double y, double z) 194 { 195 double xs, ys, zs, adj; 196 struct dd xy, r; 197 int oround; 198 int ex, ey, ez; 199 int spread; 200 201 /* 202 * Handle special cases. The order of operations and the particular 203 * return values here are crucial in handling special cases involving 204 * infinities, NaNs, overflows, and signed zeroes correctly. 205 */ 206 if (x == 0.0 || y == 0.0) 207 return (x * y + z); 208 if (z == 0.0) 209 return (x * y); 210 if (!isfinite(x) || !isfinite(y)) 211 return (x * y + z); 212 if (!isfinite(z)) 213 return (z); 214 215 xs = frexp(x, &ex); 216 ys = frexp(y, &ey); 217 zs = frexp(z, &ez); 218 oround = fegetround(); 219 spread = ex + ey - ez; 220 221 /* 222 * If x * y and z are many orders of magnitude apart, the scaling 223 * will overflow, so we handle these cases specially. Rounding 224 * modes other than FE_TONEAREST are painful. 225 */ 226 if (spread < -DBL_MANT_DIG) { 227 feraiseexcept(FE_INEXACT); 228 if (!isnormal(z)) 229 feraiseexcept(FE_UNDERFLOW); 230 switch (oround) { 231 case FE_TONEAREST: 232 return (z); 233 case FE_TOWARDZERO: 234 if ((x > 0.0) ^ (y < 0.0) ^ (z < 0.0)) 235 return (z); 236 else 237 return (nextafter(z, 0)); 238 case FE_DOWNWARD: 239 if ((x > 0.0) ^ (y < 0.0)) 240 return (z); 241 else 242 return (nextafter(z, -INFINITY)); 243 default: /* FE_UPWARD */ 244 if ((x > 0.0) ^ (y < 0.0)) 245 return (nextafter(z, INFINITY)); 246 else 247 return (z); 248 } 249 } 250 if (spread <= DBL_MANT_DIG * 2) 251 zs = ldexp(zs, -spread); 252 else 253 zs = copysign(DBL_MIN, zs); 254 255 fesetround(FE_TONEAREST); 256 /* work around clang issue #8472 */ 257 volatile double vxs = xs; 258 259 /* 260 * Basic approach for round-to-nearest: 261 * 262 * (xy.hi, xy.lo) = x * y (exact) 263 * (r.hi, r.lo) = xy.hi + z (exact) 264 * adj = xy.lo + r.lo (inexact; low bit is sticky) 265 * result = r.hi + adj (correctly rounded) 266 */ 267 xy = dd_mul(vxs, ys); 268 r = dd_add(xy.hi, zs); 269 270 spread = ex + ey; 271 272 if (r.hi == 0.0 && xy.lo == 0) { 273 /* 274 * When the addends cancel to 0, ensure that the result has 275 * the correct sign. 276 */ 277 fesetround(oround); 278 volatile double vzs = zs; /* XXX gcc CSE bug workaround */ 279 return (xy.hi + vzs); 280 } 281 282 if (oround != FE_TONEAREST) { 283 /* 284 * There is no need to worry about double rounding in directed 285 * rounding modes. 286 */ 287 fesetround(oround); 288 /* work around clang issue #8472 */ 289 volatile double vrlo = r.lo; 290 adj = vrlo + xy.lo; 291 return (ldexp(r.hi + adj, spread)); 292 } 293 294 adj = add_adjusted(r.lo, xy.lo); 295 if (spread + ilogb(r.hi) > -1023) 296 return (ldexp(r.hi + adj, spread)); 297 else 298 return (add_and_denormalize(r.hi, adj, spread)); 299 } 300 #endif /* !USE_BUILTIN_FMA */ 301 302 #if (LDBL_MANT_DIG == 53) 303 __weak_reference(fma, fmal); 304 #endif 305