1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * linux/arch/arm/vfp/vfp.h 4 * 5 * Copyright (C) 2004 ARM Limited. 6 * Written by Deep Blue Solutions Limited. 7 */ 8 9 static inline u32 vfp_shiftright32jamming(u32 val, unsigned int shift) 10 { 11 if (shift) { 12 if (shift < 32) 13 val = val >> shift | ((val << (32 - shift)) != 0); 14 else 15 val = val != 0; 16 } 17 return val; 18 } 19 20 static inline u64 vfp_shiftright64jamming(u64 val, unsigned int shift) 21 { 22 if (shift) { 23 if (shift < 64) 24 val = val >> shift | ((val << (64 - shift)) != 0); 25 else 26 val = val != 0; 27 } 28 return val; 29 } 30 31 static inline u32 vfp_hi64to32jamming(u64 val) 32 { 33 u32 v; 34 35 asm( 36 "cmp %Q1, #1 @ vfp_hi64to32jamming\n\t" 37 "movcc %0, %R1\n\t" 38 "orrcs %0, %R1, #1" 39 : "=r" (v) : "r" (val) : "cc"); 40 41 return v; 42 } 43 44 static inline void add128(u64 *resh, u64 *resl, u64 nh, u64 nl, u64 mh, u64 ml) 45 { 46 asm( "adds %Q0, %Q2, %Q4\n\t" 47 "adcs %R0, %R2, %R4\n\t" 48 "adcs %Q1, %Q3, %Q5\n\t" 49 "adc %R1, %R3, %R5" 50 : "=r" (nl), "=r" (nh) 51 : "0" (nl), "1" (nh), "r" (ml), "r" (mh) 52 : "cc"); 53 *resh = nh; 54 *resl = nl; 55 } 56 57 static inline void sub128(u64 *resh, u64 *resl, u64 nh, u64 nl, u64 mh, u64 ml) 58 { 59 asm( "subs %Q0, %Q2, %Q4\n\t" 60 "sbcs %R0, %R2, %R4\n\t" 61 "sbcs %Q1, %Q3, %Q5\n\t" 62 "sbc %R1, %R3, %R5\n\t" 63 : "=r" (nl), "=r" (nh) 64 : "0" (nl), "1" (nh), "r" (ml), "r" (mh) 65 : "cc"); 66 *resh = nh; 67 *resl = nl; 68 } 69 70 static inline void mul64to128(u64 *resh, u64 *resl, u64 n, u64 m) 71 { 72 u32 nh, nl, mh, ml; 73 u64 rh, rma, rmb, rl; 74 75 nl = n; 76 ml = m; 77 rl = (u64)nl * ml; 78 79 nh = n >> 32; 80 rma = (u64)nh * ml; 81 82 mh = m >> 32; 83 rmb = (u64)nl * mh; 84 rma += rmb; 85 86 rh = (u64)nh * mh; 87 rh += ((u64)(rma < rmb) << 32) + (rma >> 32); 88 89 rma <<= 32; 90 rl += rma; 91 rh += (rl < rma); 92 93 *resl = rl; 94 *resh = rh; 95 } 96 97 static inline void shift64left(u64 *resh, u64 *resl, u64 n) 98 { 99 *resh = n >> 63; 100 *resl = n << 1; 101 } 102 103 static inline u64 vfp_hi64multiply64(u64 n, u64 m) 104 { 105 u64 rh, rl; 106 mul64to128(&rh, &rl, n, m); 107 return rh | (rl != 0); 108 } 109 110 static inline u64 vfp_estimate_div128to64(u64 nh, u64 nl, u64 m) 111 { 112 u64 mh, ml, remh, reml, termh, terml, z; 113 114 if (nh >= m) 115 return ~0ULL; 116 mh = m >> 32; 117 if (mh << 32 <= nh) { 118 z = 0xffffffff00000000ULL; 119 } else { 120 z = nh; 121 do_div(z, mh); 122 z <<= 32; 123 } 124 mul64to128(&termh, &terml, m, z); 125 sub128(&remh, &reml, nh, nl, termh, terml); 126 ml = m << 32; 127 while ((s64)remh < 0) { 128 z -= 0x100000000ULL; 129 add128(&remh, &reml, remh, reml, mh, ml); 130 } 131 remh = (remh << 32) | (reml >> 32); 132 if (mh << 32 <= remh) { 133 z |= 0xffffffff; 134 } else { 135 do_div(remh, mh); 136 z |= remh; 137 } 138 return z; 139 } 140 141 /* 142 * Operations on unpacked elements 143 */ 144 #define vfp_sign_negate(sign) (sign ^ 0x8000) 145 146 /* 147 * Single-precision 148 */ 149 struct vfp_single { 150 s16 exponent; 151 u16 sign; 152 u32 significand; 153 }; 154 155 asmlinkage s32 vfp_get_float(unsigned int reg); 156 asmlinkage void vfp_put_float(s32 val, unsigned int reg); 157 158 /* 159 * VFP_SINGLE_MANTISSA_BITS - number of bits in the mantissa 160 * VFP_SINGLE_EXPONENT_BITS - number of bits in the exponent 161 * VFP_SINGLE_LOW_BITS - number of low bits in the unpacked significand 162 * which are not propagated to the float upon packing. 163 */ 164 #define VFP_SINGLE_MANTISSA_BITS (23) 165 #define VFP_SINGLE_EXPONENT_BITS (8) 166 #define VFP_SINGLE_LOW_BITS (32 - VFP_SINGLE_MANTISSA_BITS - 2) 167 #define VFP_SINGLE_LOW_BITS_MASK ((1 << VFP_SINGLE_LOW_BITS) - 1) 168 169 /* 170 * The bit in an unpacked float which indicates that it is a quiet NaN 171 */ 172 #define VFP_SINGLE_SIGNIFICAND_QNAN (1 << (VFP_SINGLE_MANTISSA_BITS - 1 + VFP_SINGLE_LOW_BITS)) 173 174 /* 175 * Operations on packed single-precision numbers 176 */ 177 #define vfp_single_packed_sign(v) ((v) & 0x80000000) 178 #define vfp_single_packed_negate(v) ((v) ^ 0x80000000) 179 #define vfp_single_packed_abs(v) ((v) & ~0x80000000) 180 #define vfp_single_packed_exponent(v) (((v) >> VFP_SINGLE_MANTISSA_BITS) & ((1 << VFP_SINGLE_EXPONENT_BITS) - 1)) 181 #define vfp_single_packed_mantissa(v) ((v) & ((1 << VFP_SINGLE_MANTISSA_BITS) - 1)) 182 183 /* 184 * Unpack a single-precision float. Note that this returns the magnitude 185 * of the single-precision float mantissa with the 1. if necessary, 186 * aligned to bit 30. 187 */ 188 static inline void vfp_single_unpack(struct vfp_single *s, s32 val) 189 { 190 u32 significand; 191 192 s->sign = vfp_single_packed_sign(val) >> 16, 193 s->exponent = vfp_single_packed_exponent(val); 194 195 significand = (u32) val; 196 significand = (significand << (32 - VFP_SINGLE_MANTISSA_BITS)) >> 2; 197 if (s->exponent && s->exponent != 255) 198 significand |= 0x40000000; 199 s->significand = significand; 200 } 201 202 /* 203 * Re-pack a single-precision float. This assumes that the float is 204 * already normalised such that the MSB is bit 30, _not_ bit 31. 205 */ 206 static inline s32 vfp_single_pack(struct vfp_single *s) 207 { 208 u32 val; 209 val = (s->sign << 16) + 210 (s->exponent << VFP_SINGLE_MANTISSA_BITS) + 211 (s->significand >> VFP_SINGLE_LOW_BITS); 212 return (s32)val; 213 } 214 215 #define VFP_NUMBER (1<<0) 216 #define VFP_ZERO (1<<1) 217 #define VFP_DENORMAL (1<<2) 218 #define VFP_INFINITY (1<<3) 219 #define VFP_NAN (1<<4) 220 #define VFP_NAN_SIGNAL (1<<5) 221 222 #define VFP_QNAN (VFP_NAN) 223 #define VFP_SNAN (VFP_NAN|VFP_NAN_SIGNAL) 224 225 static inline int vfp_single_type(struct vfp_single *s) 226 { 227 int type = VFP_NUMBER; 228 if (s->exponent == 255) { 229 if (s->significand == 0) 230 type = VFP_INFINITY; 231 else if (s->significand & VFP_SINGLE_SIGNIFICAND_QNAN) 232 type = VFP_QNAN; 233 else 234 type = VFP_SNAN; 235 } else if (s->exponent == 0) { 236 if (s->significand == 0) 237 type |= VFP_ZERO; 238 else 239 type |= VFP_DENORMAL; 240 } 241 return type; 242 } 243 244 #ifndef DEBUG 245 #define vfp_single_normaliseround(sd,vsd,fpscr,except,func) __vfp_single_normaliseround(sd,vsd,fpscr,except) 246 u32 __vfp_single_normaliseround(int sd, struct vfp_single *vs, u32 fpscr, u32 exceptions); 247 #else 248 u32 vfp_single_normaliseround(int sd, struct vfp_single *vs, u32 fpscr, u32 exceptions, const char *func); 249 #endif 250 251 /* 252 * Double-precision 253 */ 254 struct vfp_double { 255 s16 exponent; 256 u16 sign; 257 u64 significand; 258 }; 259 260 /* 261 * VFP_REG_ZERO is a special register number for vfp_get_double 262 * which returns (double)0.0. This is useful for the compare with 263 * zero instructions. 264 */ 265 #ifdef CONFIG_VFPv3 266 #define VFP_REG_ZERO 32 267 #else 268 #define VFP_REG_ZERO 16 269 #endif 270 asmlinkage u64 vfp_get_double(unsigned int reg); 271 asmlinkage void vfp_put_double(u64 val, unsigned int reg); 272 273 #define VFP_DOUBLE_MANTISSA_BITS (52) 274 #define VFP_DOUBLE_EXPONENT_BITS (11) 275 #define VFP_DOUBLE_LOW_BITS (64 - VFP_DOUBLE_MANTISSA_BITS - 2) 276 #define VFP_DOUBLE_LOW_BITS_MASK ((1 << VFP_DOUBLE_LOW_BITS) - 1) 277 278 /* 279 * The bit in an unpacked double which indicates that it is a quiet NaN 280 */ 281 #define VFP_DOUBLE_SIGNIFICAND_QNAN (1ULL << (VFP_DOUBLE_MANTISSA_BITS - 1 + VFP_DOUBLE_LOW_BITS)) 282 283 /* 284 * Operations on packed single-precision numbers 285 */ 286 #define vfp_double_packed_sign(v) ((v) & (1ULL << 63)) 287 #define vfp_double_packed_negate(v) ((v) ^ (1ULL << 63)) 288 #define vfp_double_packed_abs(v) ((v) & ~(1ULL << 63)) 289 #define vfp_double_packed_exponent(v) (((v) >> VFP_DOUBLE_MANTISSA_BITS) & ((1 << VFP_DOUBLE_EXPONENT_BITS) - 1)) 290 #define vfp_double_packed_mantissa(v) ((v) & ((1ULL << VFP_DOUBLE_MANTISSA_BITS) - 1)) 291 292 /* 293 * Unpack a double-precision float. Note that this returns the magnitude 294 * of the double-precision float mantissa with the 1. if necessary, 295 * aligned to bit 62. 296 */ 297 static inline void vfp_double_unpack(struct vfp_double *s, s64 val) 298 { 299 u64 significand; 300 301 s->sign = vfp_double_packed_sign(val) >> 48; 302 s->exponent = vfp_double_packed_exponent(val); 303 304 significand = (u64) val; 305 significand = (significand << (64 - VFP_DOUBLE_MANTISSA_BITS)) >> 2; 306 if (s->exponent && s->exponent != 2047) 307 significand |= (1ULL << 62); 308 s->significand = significand; 309 } 310 311 /* 312 * Re-pack a double-precision float. This assumes that the float is 313 * already normalised such that the MSB is bit 30, _not_ bit 31. 314 */ 315 static inline s64 vfp_double_pack(struct vfp_double *s) 316 { 317 u64 val; 318 val = ((u64)s->sign << 48) + 319 ((u64)s->exponent << VFP_DOUBLE_MANTISSA_BITS) + 320 (s->significand >> VFP_DOUBLE_LOW_BITS); 321 return (s64)val; 322 } 323 324 static inline int vfp_double_type(struct vfp_double *s) 325 { 326 int type = VFP_NUMBER; 327 if (s->exponent == 2047) { 328 if (s->significand == 0) 329 type = VFP_INFINITY; 330 else if (s->significand & VFP_DOUBLE_SIGNIFICAND_QNAN) 331 type = VFP_QNAN; 332 else 333 type = VFP_SNAN; 334 } else if (s->exponent == 0) { 335 if (s->significand == 0) 336 type |= VFP_ZERO; 337 else 338 type |= VFP_DENORMAL; 339 } 340 return type; 341 } 342 343 u32 vfp_double_normaliseround(int dd, struct vfp_double *vd, u32 fpscr, u32 exceptions, const char *func); 344 345 u32 vfp_estimate_sqrt_significand(u32 exponent, u32 significand); 346 347 /* 348 * A special flag to tell the normalisation code not to normalise. 349 */ 350 #define VFP_NAN_FLAG 0x100 351 352 /* 353 * A bit pattern used to indicate the initial (unset) value of the 354 * exception mask, in case nothing handles an instruction. This 355 * doesn't include the NAN flag, which get masked out before 356 * we check for an error. 357 */ 358 #define VFP_EXCEPTION_ERROR ((u32)-1 & ~VFP_NAN_FLAG) 359 360 /* 361 * A flag to tell vfp instruction type. 362 * OP_SCALAR - this operation always operates in scalar mode 363 * OP_SD - the instruction exceptionally writes to a single precision result. 364 * OP_DD - the instruction exceptionally writes to a double precision result. 365 * OP_SM - the instruction exceptionally reads from a single precision operand. 366 */ 367 #define OP_SCALAR (1 << 0) 368 #define OP_SD (1 << 1) 369 #define OP_DD (1 << 1) 370 #define OP_SM (1 << 2) 371 372 struct op { 373 u32 (* const fn)(int dd, int dn, int dm, u32 fpscr); 374 u32 flags; 375 }; 376 377 asmlinkage void vfp_save_state(void *location, u32 fpexc); 378 asmlinkage u32 vfp_load_state(const void *location); 379