1 //===-- lib/fp_lib.h - Floating-point utilities -------------------*- C -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a configuration header for soft-float routines in compiler-rt. 10 // This file does not provide any part of the compiler-rt interface, but defines 11 // many useful constants and utility routines that are used in the 12 // implementation of the soft-float routines in compiler-rt. 13 // 14 // Assumes that float, double and long double correspond to the IEEE-754 15 // binary32, binary64 and binary 128 types, respectively, and that integer 16 // endianness matches floating point endianness on the target platform. 17 // 18 //===----------------------------------------------------------------------===// 19 20 #ifndef FP_LIB_HEADER 21 #define FP_LIB_HEADER 22 23 #include "int_lib.h" 24 #include "int_math.h" 25 #include <limits.h> 26 #include <stdbool.h> 27 #include <stdint.h> 28 29 // x86_64 FreeBSD prior v9.3 define fixed-width types incorrectly in 30 // 32-bit mode. 31 #if defined(__FreeBSD__) && defined(__i386__) 32 #include <sys/param.h> 33 #if __FreeBSD_version < 903000 // v9.3 34 #define uint64_t unsigned long long 35 #define int64_t long long 36 #undef UINT64_C 37 #define UINT64_C(c) (c##ULL) 38 #endif 39 #endif 40 41 #if defined SINGLE_PRECISION 42 43 typedef uint32_t rep_t; 44 typedef int32_t srep_t; 45 typedef float fp_t; 46 #define REP_C UINT32_C 47 #define significandBits 23 48 49 static __inline int rep_clz(rep_t a) { return __builtin_clz(a); } 50 51 // 32x32 --> 64 bit multiply 52 static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { 53 const uint64_t product = (uint64_t)a * b; 54 *hi = product >> 32; 55 *lo = product; 56 } 57 COMPILER_RT_ABI fp_t __addsf3(fp_t a, fp_t b); 58 59 #elif defined DOUBLE_PRECISION 60 61 typedef uint64_t rep_t; 62 typedef int64_t srep_t; 63 typedef double fp_t; 64 #define REP_C UINT64_C 65 #define significandBits 52 66 67 static __inline int rep_clz(rep_t a) { 68 #if defined __LP64__ 69 return __builtin_clzl(a); 70 #else 71 if (a & REP_C(0xffffffff00000000)) 72 return __builtin_clz(a >> 32); 73 else 74 return 32 + __builtin_clz(a & REP_C(0xffffffff)); 75 #endif 76 } 77 78 #define loWord(a) (a & 0xffffffffU) 79 #define hiWord(a) (a >> 32) 80 81 // 64x64 -> 128 wide multiply for platforms that don't have such an operation; 82 // many 64-bit platforms have this operation, but they tend to have hardware 83 // floating-point, so we don't bother with a special case for them here. 84 static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { 85 // Each of the component 32x32 -> 64 products 86 const uint64_t plolo = loWord(a) * loWord(b); 87 const uint64_t plohi = loWord(a) * hiWord(b); 88 const uint64_t philo = hiWord(a) * loWord(b); 89 const uint64_t phihi = hiWord(a) * hiWord(b); 90 // Sum terms that contribute to lo in a way that allows us to get the carry 91 const uint64_t r0 = loWord(plolo); 92 const uint64_t r1 = hiWord(plolo) + loWord(plohi) + loWord(philo); 93 *lo = r0 + (r1 << 32); 94 // Sum terms contributing to hi with the carry from lo 95 *hi = hiWord(plohi) + hiWord(philo) + hiWord(r1) + phihi; 96 } 97 #undef loWord 98 #undef hiWord 99 100 COMPILER_RT_ABI fp_t __adddf3(fp_t a, fp_t b); 101 102 #elif defined QUAD_PRECISION 103 #if __LDBL_MANT_DIG__ == 113 && defined(__SIZEOF_INT128__) 104 #define CRT_LDBL_128BIT 105 typedef __uint128_t rep_t; 106 typedef __int128_t srep_t; 107 typedef long double fp_t; 108 #define REP_C (__uint128_t) 109 // Note: Since there is no explicit way to tell compiler the constant is a 110 // 128-bit integer, we let the constant be casted to 128-bit integer 111 #define significandBits 112 112 113 static __inline int rep_clz(rep_t a) { 114 const union { 115 __uint128_t ll; 116 #if _YUGA_BIG_ENDIAN 117 struct { 118 uint64_t high, low; 119 } s; 120 #else 121 struct { 122 uint64_t low, high; 123 } s; 124 #endif 125 } uu = {.ll = a}; 126 127 uint64_t word; 128 uint64_t add; 129 130 if (uu.s.high) { 131 word = uu.s.high; 132 add = 0; 133 } else { 134 word = uu.s.low; 135 add = 64; 136 } 137 return __builtin_clzll(word) + add; 138 } 139 140 #define Word_LoMask UINT64_C(0x00000000ffffffff) 141 #define Word_HiMask UINT64_C(0xffffffff00000000) 142 #define Word_FullMask UINT64_C(0xffffffffffffffff) 143 #define Word_1(a) (uint64_t)((a >> 96) & Word_LoMask) 144 #define Word_2(a) (uint64_t)((a >> 64) & Word_LoMask) 145 #define Word_3(a) (uint64_t)((a >> 32) & Word_LoMask) 146 #define Word_4(a) (uint64_t)(a & Word_LoMask) 147 148 // 128x128 -> 256 wide multiply for platforms that don't have such an operation; 149 // many 64-bit platforms have this operation, but they tend to have hardware 150 // floating-point, so we don't bother with a special case for them here. 151 static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { 152 153 const uint64_t product11 = Word_1(a) * Word_1(b); 154 const uint64_t product12 = Word_1(a) * Word_2(b); 155 const uint64_t product13 = Word_1(a) * Word_3(b); 156 const uint64_t product14 = Word_1(a) * Word_4(b); 157 const uint64_t product21 = Word_2(a) * Word_1(b); 158 const uint64_t product22 = Word_2(a) * Word_2(b); 159 const uint64_t product23 = Word_2(a) * Word_3(b); 160 const uint64_t product24 = Word_2(a) * Word_4(b); 161 const uint64_t product31 = Word_3(a) * Word_1(b); 162 const uint64_t product32 = Word_3(a) * Word_2(b); 163 const uint64_t product33 = Word_3(a) * Word_3(b); 164 const uint64_t product34 = Word_3(a) * Word_4(b); 165 const uint64_t product41 = Word_4(a) * Word_1(b); 166 const uint64_t product42 = Word_4(a) * Word_2(b); 167 const uint64_t product43 = Word_4(a) * Word_3(b); 168 const uint64_t product44 = Word_4(a) * Word_4(b); 169 170 const __uint128_t sum0 = (__uint128_t)product44; 171 const __uint128_t sum1 = (__uint128_t)product34 + (__uint128_t)product43; 172 const __uint128_t sum2 = 173 (__uint128_t)product24 + (__uint128_t)product33 + (__uint128_t)product42; 174 const __uint128_t sum3 = (__uint128_t)product14 + (__uint128_t)product23 + 175 (__uint128_t)product32 + (__uint128_t)product41; 176 const __uint128_t sum4 = 177 (__uint128_t)product13 + (__uint128_t)product22 + (__uint128_t)product31; 178 const __uint128_t sum5 = (__uint128_t)product12 + (__uint128_t)product21; 179 const __uint128_t sum6 = (__uint128_t)product11; 180 181 const __uint128_t r0 = (sum0 & Word_FullMask) + ((sum1 & Word_LoMask) << 32); 182 const __uint128_t r1 = (sum0 >> 64) + ((sum1 >> 32) & Word_FullMask) + 183 (sum2 & Word_FullMask) + ((sum3 << 32) & Word_HiMask); 184 185 *lo = r0 + (r1 << 64); 186 *hi = (r1 >> 64) + (sum1 >> 96) + (sum2 >> 64) + (sum3 >> 32) + sum4 + 187 (sum5 << 32) + (sum6 << 64); 188 } 189 #undef Word_1 190 #undef Word_2 191 #undef Word_3 192 #undef Word_4 193 #undef Word_HiMask 194 #undef Word_LoMask 195 #undef Word_FullMask 196 #endif // __LDBL_MANT_DIG__ == 113 && __SIZEOF_INT128__ 197 #else 198 #error SINGLE_PRECISION, DOUBLE_PRECISION or QUAD_PRECISION must be defined. 199 #endif 200 201 #if defined(SINGLE_PRECISION) || defined(DOUBLE_PRECISION) || \ 202 defined(CRT_LDBL_128BIT) 203 #define typeWidth (sizeof(rep_t) * CHAR_BIT) 204 #define exponentBits (typeWidth - significandBits - 1) 205 #define maxExponent ((1 << exponentBits) - 1) 206 #define exponentBias (maxExponent >> 1) 207 208 #define implicitBit (REP_C(1) << significandBits) 209 #define significandMask (implicitBit - 1U) 210 #define signBit (REP_C(1) << (significandBits + exponentBits)) 211 #define absMask (signBit - 1U) 212 #define exponentMask (absMask ^ significandMask) 213 #define oneRep ((rep_t)exponentBias << significandBits) 214 #define infRep exponentMask 215 #define quietBit (implicitBit >> 1) 216 #define qnanRep (exponentMask | quietBit) 217 218 static __inline rep_t toRep(fp_t x) { 219 const union { 220 fp_t f; 221 rep_t i; 222 } rep = {.f = x}; 223 return rep.i; 224 } 225 226 static __inline fp_t fromRep(rep_t x) { 227 const union { 228 fp_t f; 229 rep_t i; 230 } rep = {.i = x}; 231 return rep.f; 232 } 233 234 static __inline int normalize(rep_t *significand) { 235 const int shift = rep_clz(*significand) - rep_clz(implicitBit); 236 *significand <<= shift; 237 return 1 - shift; 238 } 239 240 static __inline void wideLeftShift(rep_t *hi, rep_t *lo, int count) { 241 *hi = *hi << count | *lo >> (typeWidth - count); 242 *lo = *lo << count; 243 } 244 245 static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo, 246 unsigned int count) { 247 if (count < typeWidth) { 248 const bool sticky = *lo << (typeWidth - count); 249 *lo = *hi << (typeWidth - count) | *lo >> count | sticky; 250 *hi = *hi >> count; 251 } else if (count < 2 * typeWidth) { 252 const bool sticky = *hi << (2 * typeWidth - count) | *lo; 253 *lo = *hi >> (count - typeWidth) | sticky; 254 *hi = 0; 255 } else { 256 const bool sticky = *hi | *lo; 257 *lo = sticky; 258 *hi = 0; 259 } 260 } 261 262 // Implements logb methods (logb, logbf, logbl) for IEEE-754. This avoids 263 // pulling in a libm dependency from compiler-rt, but is not meant to replace 264 // it (i.e. code calling logb() should get the one from libm, not this), hence 265 // the __compiler_rt prefix. 266 static __inline fp_t __compiler_rt_logbX(fp_t x) { 267 rep_t rep = toRep(x); 268 int exp = (rep & exponentMask) >> significandBits; 269 270 // Abnormal cases: 271 // 1) +/- inf returns +inf; NaN returns NaN 272 // 2) 0.0 returns -inf 273 if (exp == maxExponent) { 274 if (((rep & signBit) == 0) || (x != x)) { 275 return x; // NaN or +inf: return x 276 } else { 277 return -x; // -inf: return -x 278 } 279 } else if (x == 0.0) { 280 // 0.0: return -inf 281 return fromRep(infRep | signBit); 282 } 283 284 if (exp != 0) { 285 // Normal number 286 return exp - exponentBias; // Unbias exponent 287 } else { 288 // Subnormal number; normalize and repeat 289 rep &= absMask; 290 const int shift = 1 - normalize(&rep); 291 exp = (rep & exponentMask) >> significandBits; 292 return exp - exponentBias - shift; // Unbias exponent 293 } 294 } 295 #endif 296 297 #if defined(SINGLE_PRECISION) 298 static __inline fp_t __compiler_rt_logbf(fp_t x) { 299 return __compiler_rt_logbX(x); 300 } 301 #elif defined(DOUBLE_PRECISION) 302 static __inline fp_t __compiler_rt_logb(fp_t x) { 303 return __compiler_rt_logbX(x); 304 } 305 #elif defined(QUAD_PRECISION) 306 #if defined(CRT_LDBL_128BIT) 307 static __inline fp_t __compiler_rt_logbl(fp_t x) { 308 return __compiler_rt_logbX(x); 309 } 310 #else 311 // The generic implementation only works for ieee754 floating point. For other 312 // floating point types, continue to rely on the libm implementation for now. 313 static __inline long double __compiler_rt_logbl(long double x) { 314 return crt_logbl(x); 315 } 316 #endif 317 #endif 318 319 #endif // FP_LIB_HEADER 320