1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24 */ 25 /* 26 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 27 * Use is subject to license terms. 28 */ 29 30 #if defined(ELFOBJ) 31 #pragma weak fma = __fma 32 #endif 33 34 #include "libm.h" 35 #include "fma.h" 36 #include "fenv_inlines.h" 37 38 #if defined(__sparc) 39 40 static const union { 41 unsigned i[2]; 42 double d; 43 } C[] = { 44 { 0x3fe00000u, 0 }, 45 { 0x40000000u, 0 }, 46 { 0x43300000u, 0 }, 47 { 0x41a00000u, 0 }, 48 { 0x3e500000u, 0 }, 49 { 0x3df00000u, 0 }, 50 { 0x3bf00000u, 0 }, 51 { 0x7fe00000u, 0 }, 52 { 0x00100000u, 0 }, 53 { 0x00100001u, 0 } 54 }; 55 56 #define half C[0].d 57 #define two C[1].d 58 #define two52 C[2].d 59 #define two27 C[3].d 60 #define twom26 C[4].d 61 #define twom32 C[5].d 62 #define twom64 C[6].d 63 #define huge C[7].d 64 #define tiny C[8].d 65 #define tiny2 C[9].d 66 67 static const unsigned int fsr_rm = 0xc0000000u; 68 69 /* 70 * fma for SPARC: 64-bit double precision, big-endian 71 */ 72 double 73 __fma(double x, double y, double z) { 74 union { 75 unsigned i[2]; 76 double d; 77 } xx, yy, zz; 78 double xhi, yhi, xlo, ylo, t; 79 unsigned int xy0, xy1, xy2, xy3, z0, z1, z2, z3, fsr, rm, sticky; 80 int hx, hy, hz, ex, ey, ez, exy, sxy, sz, e, ibit; 81 volatile double dummy; 82 83 /* extract the high order words of the arguments */ 84 xx.d = x; 85 yy.d = y; 86 zz.d = z; 87 hx = xx.i[0] & ~0x80000000; 88 hy = yy.i[0] & ~0x80000000; 89 hz = zz.i[0] & ~0x80000000; 90 91 /* dispense with inf, nan, and zero cases */ 92 if (hx >= 0x7ff00000 || hy >= 0x7ff00000 || (hx | xx.i[1]) == 0 || 93 (hy | yy.i[1]) == 0) /* x or y is inf, nan, or zero */ 94 return (x * y + z); 95 96 if (hz >= 0x7ff00000) /* z is inf or nan */ 97 return (x + z); /* avoid spurious under/overflow in x * y */ 98 99 if ((hz | zz.i[1]) == 0) /* z is zero */ 100 /* 101 * x * y isn't zero but could underflow to zero, 102 * so don't add z, lest we perturb the sign 103 */ 104 return (x * y); 105 106 /* 107 * now x, y, and z are all finite and nonzero; save the fsr and 108 * set round-to-negative-infinity mode (and clear nonstandard 109 * mode before we try to scale subnormal operands) 110 */ 111 __fenv_getfsr32(&fsr); 112 __fenv_setfsr32(&fsr_rm); 113 114 /* extract signs and exponents, and normalize subnormals */ 115 sxy = (xx.i[0] ^ yy.i[0]) & 0x80000000; 116 sz = zz.i[0] & 0x80000000; 117 ex = hx >> 20; 118 if (!ex) { 119 xx.d = x * two52; 120 ex = ((xx.i[0] & ~0x80000000) >> 20) - 52; 121 } 122 ey = hy >> 20; 123 if (!ey) { 124 yy.d = y * two52; 125 ey = ((yy.i[0] & ~0x80000000) >> 20) - 52; 126 } 127 ez = hz >> 20; 128 if (!ez) { 129 zz.d = z * two52; 130 ez = ((zz.i[0] & ~0x80000000) >> 20) - 52; 131 } 132 133 /* multiply x*y to 106 bits */ 134 exy = ex + ey - 0x3ff; 135 xx.i[0] = (xx.i[0] & 0xfffff) | 0x3ff00000; 136 yy.i[0] = (yy.i[0] & 0xfffff) | 0x3ff00000; 137 x = xx.d; 138 y = yy.d; 139 xhi = ((x + twom26) + two27) - two27; 140 yhi = ((y + twom26) + two27) - two27; 141 xlo = x - xhi; 142 ylo = y - yhi; 143 x *= y; 144 y = ((xhi * yhi - x) + xhi * ylo + xlo * yhi) + xlo * ylo; 145 if (x >= two) { 146 x *= half; 147 y *= half; 148 exy++; 149 } 150 151 /* extract the significands */ 152 xx.d = x; 153 xy0 = (xx.i[0] & 0xfffff) | 0x100000; 154 xy1 = xx.i[1]; 155 yy.d = t = y + twom32; 156 xy2 = yy.i[1]; 157 yy.d = (y - (t - twom32)) + twom64; 158 xy3 = yy.i[1]; 159 z0 = (zz.i[0] & 0xfffff) | 0x100000; 160 z1 = zz.i[1]; 161 z2 = z3 = 0; 162 163 /* 164 * now x*y is represented by sxy, exy, and xy[0-3], and z is 165 * represented likewise; swap if need be so |xy| <= |z| 166 */ 167 if (exy > ez || (exy == ez && (xy0 > z0 || (xy0 == z0 && 168 (xy1 > z1 || (xy1 == z1 && (xy2 | xy3) != 0)))))) { 169 e = sxy; sxy = sz; sz = e; 170 e = exy; exy = ez; ez = e; 171 e = xy0; xy0 = z0; z0 = e; 172 e = xy1; xy1 = z1; z1 = e; 173 z2 = xy2; xy2 = 0; 174 z3 = xy3; xy3 = 0; 175 } 176 177 /* shift the significand of xy keeping a sticky bit */ 178 e = ez - exy; 179 if (e > 116) { 180 xy0 = xy1 = xy2 = 0; 181 xy3 = 1; 182 } else if (e >= 96) { 183 sticky = xy3 | xy2 | xy1 | ((xy0 << 1) << (127 - e)); 184 xy3 = xy0 >> (e - 96); 185 if (sticky) 186 xy3 |= 1; 187 xy0 = xy1 = xy2 = 0; 188 } else if (e >= 64) { 189 sticky = xy3 | xy2 | ((xy1 << 1) << (95 - e)); 190 xy3 = (xy1 >> (e - 64)) | ((xy0 << 1) << (95 - e)); 191 if (sticky) 192 xy3 |= 1; 193 xy2 = xy0 >> (e - 64); 194 xy0 = xy1 = 0; 195 } else if (e >= 32) { 196 sticky = xy3 | ((xy2 << 1) << (63 - e)); 197 xy3 = (xy2 >> (e - 32)) | ((xy1 << 1) << (63 - e)); 198 if (sticky) 199 xy3 |= 1; 200 xy2 = (xy1 >> (e - 32)) | ((xy0 << 1) << (63 - e)); 201 xy1 = xy0 >> (e - 32); 202 xy0 = 0; 203 } else if (e) { 204 sticky = (xy3 << 1) << (31 - e); 205 xy3 = (xy3 >> e) | ((xy2 << 1) << (31 - e)); 206 if (sticky) 207 xy3 |= 1; 208 xy2 = (xy2 >> e) | ((xy1 << 1) << (31 - e)); 209 xy1 = (xy1 >> e) | ((xy0 << 1) << (31 - e)); 210 xy0 >>= e; 211 } 212 213 /* if this is a magnitude subtract, negate the significand of xy */ 214 if (sxy ^ sz) { 215 xy0 = ~xy0; 216 xy1 = ~xy1; 217 xy2 = ~xy2; 218 xy3 = -xy3; 219 if (xy3 == 0) 220 if (++xy2 == 0) 221 if (++xy1 == 0) 222 xy0++; 223 } 224 225 /* add, propagating carries */ 226 z3 += xy3; 227 e = (z3 < xy3); 228 z2 += xy2; 229 if (e) { 230 z2++; 231 e = (z2 <= xy2); 232 } else 233 e = (z2 < xy2); 234 z1 += xy1; 235 if (e) { 236 z1++; 237 e = (z1 <= xy1); 238 } else 239 e = (z1 < xy1); 240 z0 += xy0; 241 if (e) 242 z0++; 243 244 /* postnormalize and collect rounding information into z2 */ 245 if (ez < 1) { 246 /* result is tiny; shift right until exponent is within range */ 247 e = 1 - ez; 248 if (e > 56) { 249 z2 = 1; /* result can't be exactly zero */ 250 z0 = z1 = 0; 251 } else if (e >= 32) { 252 sticky = z3 | z2 | ((z1 << 1) << (63 - e)); 253 z2 = (z1 >> (e - 32)) | ((z0 << 1) << (63 - e)); 254 if (sticky) 255 z2 |= 1; 256 z1 = z0 >> (e - 32); 257 z0 = 0; 258 } else { 259 sticky = z3 | (z2 << 1) << (31 - e); 260 z2 = (z2 >> e) | ((z1 << 1) << (31 - e)); 261 if (sticky) 262 z2 |= 1; 263 z1 = (z1 >> e) | ((z0 << 1) << (31 - e)); 264 z0 >>= e; 265 } 266 ez = 1; 267 } else if (z0 >= 0x200000) { 268 /* carry out; shift right by one */ 269 sticky = (z2 & 1) | z3; 270 z2 = (z2 >> 1) | (z1 << 31); 271 if (sticky) 272 z2 |= 1; 273 z1 = (z1 >> 1) | (z0 << 31); 274 z0 >>= 1; 275 ez++; 276 } else { 277 if (z0 < 0x100000 && (z0 | z1 | z2 | z3) != 0) { 278 /* 279 * borrow/cancellation; shift left as much as 280 * exponent allows 281 */ 282 while (!(z0 | (z1 & 0xffe00000)) && ez >= 33) { 283 z0 = z1; 284 z1 = z2; 285 z2 = z3; 286 z3 = 0; 287 ez -= 32; 288 } 289 while (z0 < 0x100000 && ez > 1) { 290 z0 = (z0 << 1) | (z1 >> 31); 291 z1 = (z1 << 1) | (z2 >> 31); 292 z2 = (z2 << 1) | (z3 >> 31); 293 z3 <<= 1; 294 ez--; 295 } 296 } 297 if (z3) 298 z2 |= 1; 299 } 300 301 /* get the rounding mode and clear current exceptions */ 302 rm = fsr >> 30; 303 fsr &= ~FSR_CEXC; 304 305 /* strip off the integer bit, if there is one */ 306 ibit = z0 & 0x100000; 307 if (ibit) 308 z0 -= 0x100000; 309 else { 310 ez = 0; 311 if (!(z0 | z1 | z2)) { /* exact zero */ 312 zz.i[0] = rm == FSR_RM ? 0x80000000 : 0; 313 zz.i[1] = 0; 314 __fenv_setfsr32(&fsr); 315 return (zz.d); 316 } 317 } 318 319 /* 320 * flip the sense of directed roundings if the result is negative; 321 * the logic below applies to a positive result 322 */ 323 if (sz) 324 rm ^= rm >> 1; 325 326 /* round and raise exceptions */ 327 if (z2) { 328 fsr |= FSR_NXC; 329 330 /* decide whether to round the fraction up */ 331 if (rm == FSR_RP || (rm == FSR_RN && (z2 > 0x80000000u || 332 (z2 == 0x80000000u && (z1 & 1))))) { 333 /* round up and renormalize if necessary */ 334 if (++z1 == 0) { 335 if (++z0 == 0x100000) { 336 z0 = 0; 337 ez++; 338 } 339 } 340 } 341 } 342 343 /* check for under/overflow */ 344 if (ez >= 0x7ff) { 345 if (rm == FSR_RN || rm == FSR_RP) { 346 zz.i[0] = sz | 0x7ff00000; 347 zz.i[1] = 0; 348 } else { 349 zz.i[0] = sz | 0x7fefffff; 350 zz.i[1] = 0xffffffff; 351 } 352 fsr |= FSR_OFC | FSR_NXC; 353 } else { 354 zz.i[0] = sz | (ez << 20) | z0; 355 zz.i[1] = z1; 356 357 /* 358 * !ibit => exact result was tiny before rounding, 359 * z2 nonzero => result delivered is inexact 360 */ 361 if (!ibit) { 362 if (z2) 363 fsr |= FSR_UFC | FSR_NXC; 364 else if (fsr & FSR_UFM) 365 fsr |= FSR_UFC; 366 } 367 } 368 369 /* restore the fsr and emulate exceptions as needed */ 370 if ((fsr & FSR_CEXC) & (fsr >> 23)) { 371 __fenv_setfsr32(&fsr); 372 if (fsr & FSR_OFC) { 373 dummy = huge; 374 dummy *= huge; 375 } else if (fsr & FSR_UFC) { 376 dummy = tiny; 377 if (fsr & FSR_NXC) 378 dummy *= tiny; 379 else 380 dummy -= tiny2; 381 } else { 382 dummy = huge; 383 dummy += tiny; 384 } 385 } else { 386 fsr |= (fsr & 0x1f) << 5; 387 __fenv_setfsr32(&fsr); 388 } 389 return (zz.d); 390 } 391 392 #elif defined(__x86) 393 394 #if defined(__amd64) 395 #define NI 4 396 #else 397 #define NI 3 398 #endif 399 400 /* 401 * fma for x86: 64-bit double precision, little-endian 402 */ 403 double 404 __fma(double x, double y, double z) { 405 union { 406 unsigned i[NI]; 407 long double e; 408 } xx, yy, zz; 409 long double xe, ye, xhi, xlo, yhi, ylo; 410 int ex, ey, ez; 411 unsigned cwsw, oldcwsw, rm; 412 413 /* convert the operands to double extended */ 414 xx.e = (long double) x; 415 yy.e = (long double) y; 416 zz.e = (long double) z; 417 418 /* extract the exponents of the arguments */ 419 ex = xx.i[2] & 0x7fff; 420 ey = yy.i[2] & 0x7fff; 421 ez = zz.i[2] & 0x7fff; 422 423 /* dispense with inf, nan, and zero cases */ 424 if (ex == 0x7fff || ey == 0x7fff || ex == 0 || ey == 0) 425 /* x or y is inf, nan, or zero */ 426 return ((double) (xx.e * yy.e + zz.e)); 427 428 if (ez >= 0x7fff) /* z is inf or nan */ 429 return ((double) (xx.e + zz.e)); 430 /* avoid spurious inexact in x * y */ 431 432 /* 433 * save the control and status words, mask all exceptions, and 434 * set rounding to 64-bit precision and to-nearest 435 */ 436 __fenv_getcwsw(&oldcwsw); 437 cwsw = (oldcwsw & 0xf0c0ffff) | 0x033f0000; 438 __fenv_setcwsw(&cwsw); 439 440 /* multiply x*y to 106 bits */ 441 xe = xx.e; 442 xx.i[0] = 0; 443 xhi = xx.e; /* hi 32 bits */ 444 xlo = xe - xhi; /* lo 21 bits */ 445 ye = yy.e; 446 yy.i[0] = 0; 447 yhi = yy.e; 448 ylo = ye - yhi; 449 xe = xe * ye; 450 ye = ((xhi * yhi - xe) + xhi * ylo + xlo * yhi) + xlo * ylo; 451 452 /* distill the sum of xe, ye, and z */ 453 xhi = ye + zz.e; 454 yhi = xhi - ye; 455 xlo = (zz.e - yhi) + (ye - (xhi - yhi)); 456 /* now (xhi,xlo) = ye + z */ 457 458 yhi = xe + xhi; 459 ye = yhi - xe; 460 ylo = (xhi - ye) + (xe - (yhi - ye)); /* now (yhi,ylo) = xe + xhi */ 461 462 xhi = xlo + ylo; 463 xe = xhi - xlo; 464 xlo = (ylo - xe) + (xlo - (xhi - xe)); /* now (xhi,xlo) = xlo + ylo */ 465 466 yy.e = yhi + xhi; 467 ylo = (yhi - yy.e) + xhi; /* now (yy.e,ylo) = xhi + yhi */ 468 469 if (yy.i[1] != 0) { /* yy.e is nonzero */ 470 /* perturb yy.e if its least significant 10 bits are zero */ 471 if (!(yy.i[0] & 0x3ff)) { 472 xx.e = ylo + xlo; 473 if (xx.i[1] != 0) { 474 xx.i[2] = (xx.i[2] & 0x8000) | 475 ((yy.i[2] & 0x7fff) - 63); 476 xx.i[1] = 0x80000000; 477 xx.i[0] = 0; 478 yy.e += xx.e; 479 } 480 } 481 } else { 482 /* set sign of zero result according to rounding direction */ 483 rm = oldcwsw & 0x0c000000; 484 yy.i[2] = ((rm == FCW_RM)? 0x8000 : 0); 485 } 486 487 /* 488 * restore the control and status words and convert the result 489 * to double 490 */ 491 __fenv_setcwsw(&oldcwsw); 492 return ((double) yy.e); 493 } 494 495 #else 496 #error Unknown architecture 497 #endif 498