1 //===-- x86_64 floating point env manipulation functions --------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FENVIMPL_H
10 #define LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FENVIMPL_H
11
12 #include "src/__support/macros/attributes.h" // LIBC_INLINE
13 #include "src/__support/macros/config.h"
14 #include "src/__support/macros/properties/architectures.h"
15
16 #if !defined(LIBC_TARGET_ARCH_IS_X86)
17 #error "Invalid include"
18 #endif
19
20 #include <stdint.h>
21
22 #include "hdr/types/fenv_t.h"
23 #include "src/__support/macros/sanitizer.h"
24
25 namespace LIBC_NAMESPACE_DECL {
26 namespace fputil {
27
28 namespace internal {
29
30 // Normally, one should be able to define FE_* macros to the exact rounding mode
31 // encodings. However, since we want LLVM libc to be compiled against headers
32 // from other libcs, we cannot assume that FE_* macros are always defined in
33 // such a manner. So, we will define enums corresponding to the x86_64 bit
34 // encodings. The implementations can map from FE_* to the corresponding enum
35 // values.
36
37 // The rounding control values in the x87 control register and the MXCSR
38 // register have the same 2-bit enoding but have different bit positions.
39 // See below for the bit positions.
40 struct RoundingControlValue {
41 static constexpr uint16_t TO_NEAREST = 0x0;
42 static constexpr uint16_t DOWNWARD = 0x1;
43 static constexpr uint16_t UPWARD = 0x2;
44 static constexpr uint16_t TOWARD_ZERO = 0x3;
45 };
46
47 static constexpr uint16_t X87_ROUNDING_CONTROL_BIT_POSITION = 10;
48 static constexpr uint16_t MXCSR_ROUNDING_CONTROL_BIT_POSITION = 13;
49
50 // The exception flags in the x87 status register and the MXCSR have the same
51 // encoding as well as the same bit positions.
52 struct ExceptionFlags {
53 static constexpr uint16_t INVALID_F = 0x1;
54 // Some libcs define __FE_DENORM corresponding to the denormal input
55 // exception and include it in FE_ALL_EXCEPTS. We define and use it to
56 // support compiling against headers provided by such libcs.
57 static constexpr uint16_t DENORMAL_F = 0x2;
58 static constexpr uint16_t DIV_BY_ZERO_F = 0x4;
59 static constexpr uint16_t OVERFLOW_F = 0x8;
60 static constexpr uint16_t UNDERFLOW_F = 0x10;
61 static constexpr uint16_t INEXACT_F = 0x20;
62 };
63
64 // The exception control bits occupy six bits, one bit for each exception.
65 // In the x87 control word, they occupy the first 6 bits. In the MXCSR
66 // register, they occupy bits 7 to 12.
67 static constexpr uint16_t X87_EXCEPTION_CONTROL_BIT_POSITION = 0;
68 static constexpr uint16_t X87_EXCEPTION_CONTROL_BIT_POSITION_HIGH = 24;
69 static constexpr uint16_t MXCSR_EXCEPTION_CONTOL_BIT_POISTION = 7;
70
71 // Exception flags are individual bits in the corresponding registers.
72 // So, we just OR the bit values to get the full set of exceptions.
get_status_value_for_except(int excepts)73 LIBC_INLINE uint16_t get_status_value_for_except(int excepts) {
74 // We will make use of the fact that exception control bits are single
75 // bit flags in the control registers.
76 return ((excepts & FE_INVALID) ? ExceptionFlags::INVALID_F : 0) |
77 #ifdef __FE_DENORM
78 ((excepts & __FE_DENORM) ? ExceptionFlags::DENORMAL_F : 0) |
79 #endif // __FE_DENORM
80 ((excepts & FE_DIVBYZERO) ? ExceptionFlags::DIV_BY_ZERO_F : 0) |
81 ((excepts & FE_OVERFLOW) ? ExceptionFlags::OVERFLOW_F : 0) |
82 ((excepts & FE_UNDERFLOW) ? ExceptionFlags::UNDERFLOW_F : 0) |
83 ((excepts & FE_INEXACT) ? ExceptionFlags::INEXACT_F : 0);
84 }
85
exception_status_to_macro(uint16_t status)86 LIBC_INLINE int exception_status_to_macro(uint16_t status) {
87 return ((status & ExceptionFlags::INVALID_F) ? FE_INVALID : 0) |
88 #ifdef __FE_DENORM
89 ((status & ExceptionFlags::DENORMAL_F) ? __FE_DENORM : 0) |
90 #endif // __FE_DENORM
91 ((status & ExceptionFlags::DIV_BY_ZERO_F) ? FE_DIVBYZERO : 0) |
92 ((status & ExceptionFlags::OVERFLOW_F) ? FE_OVERFLOW : 0) |
93 ((status & ExceptionFlags::UNDERFLOW_F) ? FE_UNDERFLOW : 0) |
94 ((status & ExceptionFlags::INEXACT_F) ? FE_INEXACT : 0);
95 }
96
97 struct X87StateDescriptor {
98 uint16_t control_word;
99 uint16_t unused1;
100 uint16_t status_word;
101 uint16_t unused2;
102 // TODO: Elaborate the remaining 20 bytes as required.
103 uint32_t _[5];
104 };
105
get_x87_control_word()106 LIBC_INLINE uint16_t get_x87_control_word() {
107 uint16_t w;
108 __asm__ __volatile__("fnstcw %0" : "=m"(w)::);
109 MSAN_UNPOISON(&w, sizeof(w));
110 return w;
111 }
112
write_x87_control_word(uint16_t w)113 LIBC_INLINE void write_x87_control_word(uint16_t w) {
114 __asm__ __volatile__("fldcw %0" : : "m"(w) :);
115 }
116
get_x87_status_word()117 LIBC_INLINE uint16_t get_x87_status_word() {
118 uint16_t w;
119 __asm__ __volatile__("fnstsw %0" : "=m"(w)::);
120 MSAN_UNPOISON(&w, sizeof(w));
121 return w;
122 }
123
clear_x87_exceptions()124 LIBC_INLINE void clear_x87_exceptions() {
125 __asm__ __volatile__("fnclex" : : :);
126 }
127
get_mxcsr()128 LIBC_INLINE uint32_t get_mxcsr() {
129 uint32_t w;
130 __asm__ __volatile__("stmxcsr %0" : "=m"(w)::);
131 MSAN_UNPOISON(&w, sizeof(w));
132 return w;
133 }
134
write_mxcsr(uint32_t w)135 LIBC_INLINE void write_mxcsr(uint32_t w) {
136 __asm__ __volatile__("ldmxcsr %0" : : "m"(w) :);
137 }
138
get_x87_state_descriptor(X87StateDescriptor & s)139 LIBC_INLINE void get_x87_state_descriptor(X87StateDescriptor &s) {
140 __asm__ __volatile__("fnstenv %0" : "=m"(s));
141 MSAN_UNPOISON(&s, sizeof(s));
142 }
143
write_x87_state_descriptor(const X87StateDescriptor & s)144 LIBC_INLINE void write_x87_state_descriptor(const X87StateDescriptor &s) {
145 __asm__ __volatile__("fldenv %0" : : "m"(s) :);
146 }
147
fwait()148 LIBC_INLINE void fwait() { __asm__ __volatile__("fwait"); }
149
150 } // namespace internal
151
enable_except(int excepts)152 LIBC_INLINE int enable_except(int excepts) {
153 // In the x87 control word and in MXCSR, an exception is blocked
154 // if the corresponding bit is set. That is the reason for all the
155 // bit-flip operations below as we need to turn the bits to zero
156 // to enable them.
157
158 uint16_t bit_mask = internal::get_status_value_for_except(excepts);
159
160 uint16_t x87_cw = internal::get_x87_control_word();
161 uint16_t old_excepts = ~x87_cw & 0x3F; // Save previously enabled exceptions.
162 x87_cw &= ~bit_mask;
163 internal::write_x87_control_word(x87_cw);
164
165 // Enabling SSE exceptions via MXCSR is a nice thing to do but
166 // might not be of much use practically as SSE exceptions and the x87
167 // exceptions are independent of each other.
168 uint32_t mxcsr = internal::get_mxcsr();
169 mxcsr &= ~(bit_mask << internal::MXCSR_EXCEPTION_CONTOL_BIT_POISTION);
170 internal::write_mxcsr(mxcsr);
171
172 // Since the x87 exceptions and SSE exceptions are independent of each,
173 // it doesn't make much sence to report both in the return value. Most
174 // often, the standard floating point functions deal with FPU operations
175 // so we will retrun only the old x87 exceptions.
176 return internal::exception_status_to_macro(old_excepts);
177 }
178
disable_except(int excepts)179 LIBC_INLINE int disable_except(int excepts) {
180 // In the x87 control word and in MXCSR, an exception is blocked
181 // if the corresponding bit is set.
182
183 uint16_t bit_mask = internal::get_status_value_for_except(excepts);
184
185 uint16_t x87_cw = internal::get_x87_control_word();
186 uint16_t old_excepts = ~x87_cw & 0x3F; // Save previously enabled exceptions.
187 x87_cw |= bit_mask;
188 internal::write_x87_control_word(x87_cw);
189
190 // Just like in enable_except, it is not clear if disabling SSE exceptions
191 // is required. But, we will still do it only as a "nice thing to do".
192 uint32_t mxcsr = internal::get_mxcsr();
193 mxcsr |= (bit_mask << internal::MXCSR_EXCEPTION_CONTOL_BIT_POISTION);
194 internal::write_mxcsr(mxcsr);
195
196 return internal::exception_status_to_macro(old_excepts);
197 }
198
get_except()199 LIBC_INLINE int get_except() {
200 uint16_t mxcsr = static_cast<uint16_t>(internal::get_mxcsr());
201 uint16_t enabled_excepts = ~(mxcsr >> 7) & 0x3F;
202 return internal::exception_status_to_macro(enabled_excepts);
203 }
204
clear_except(int excepts)205 LIBC_INLINE int clear_except(int excepts) {
206 internal::X87StateDescriptor state;
207 internal::get_x87_state_descriptor(state);
208 state.status_word &=
209 static_cast<uint16_t>(~internal::get_status_value_for_except(excepts));
210 internal::write_x87_state_descriptor(state);
211
212 uint32_t mxcsr = internal::get_mxcsr();
213 mxcsr &= ~internal::get_status_value_for_except(excepts);
214 internal::write_mxcsr(mxcsr);
215 return 0;
216 }
217
test_except(int excepts)218 LIBC_INLINE int test_except(int excepts) {
219 uint16_t status_word = internal::get_x87_status_word();
220 uint32_t mxcsr = internal::get_mxcsr();
221 // Check both x87 status word and MXCSR.
222 uint16_t status_value = internal::get_status_value_for_except(excepts);
223 return internal::exception_status_to_macro(
224 static_cast<uint16_t>(status_value & (status_word | mxcsr)));
225 }
226
227 // Sets the exception flags but does not trigger the exception handler.
set_except(int excepts)228 LIBC_INLINE int set_except(int excepts) {
229 uint16_t status_value = internal::get_status_value_for_except(excepts);
230 internal::X87StateDescriptor state;
231 internal::get_x87_state_descriptor(state);
232 state.status_word |= status_value;
233 internal::write_x87_state_descriptor(state);
234
235 uint32_t mxcsr = internal::get_mxcsr();
236 mxcsr |= status_value;
237 internal::write_mxcsr(mxcsr);
238
239 return 0;
240 }
241
raise_except(int excepts)242 template <bool SKIP_X87_FPU = false> LIBC_INLINE int raise_except(int excepts) {
243 uint16_t status_value = internal::get_status_value_for_except(excepts);
244
245 // We set the status flag for exception one at a time and call the
246 // fwait instruction to actually get the processor to raise the
247 // exception by calling the exception handler. This scheme is per
248 // the description in "8.6 X87 FPU EXCEPTION SYNCHRONIZATION"
249 // of the "Intel 64 and IA-32 Architectures Software Developer's
250 // Manual, Vol 1".
251
252 // FPU status word is read for each exception separately as the
253 // exception handler can potentially write to it (typically to clear
254 // the corresponding exception flag). By reading it separately, we
255 // ensure that the writes by the exception handler are maintained
256 // when raising the next exception.
257
258 auto raise_helper = [](uint16_t singleExceptFlag) {
259 if constexpr (!SKIP_X87_FPU) {
260 internal::X87StateDescriptor state;
261 internal::get_x87_state_descriptor(state);
262 state.status_word |= singleExceptFlag;
263 internal::write_x87_state_descriptor(state);
264 }
265
266 uint32_t mxcsr = 0;
267 mxcsr = internal::get_mxcsr();
268 mxcsr |= singleExceptFlag;
269 internal::write_mxcsr(mxcsr);
270 internal::fwait();
271 };
272
273 if (status_value & internal::ExceptionFlags::INVALID_F)
274 raise_helper(internal::ExceptionFlags::INVALID_F);
275 if (status_value & internal::ExceptionFlags::DIV_BY_ZERO_F)
276 raise_helper(internal::ExceptionFlags::DIV_BY_ZERO_F);
277 if (status_value & internal::ExceptionFlags::OVERFLOW_F)
278 raise_helper(internal::ExceptionFlags::OVERFLOW_F);
279 if (status_value & internal::ExceptionFlags::UNDERFLOW_F)
280 raise_helper(internal::ExceptionFlags::UNDERFLOW_F);
281 if (status_value & internal::ExceptionFlags::INEXACT_F)
282 raise_helper(internal::ExceptionFlags::INEXACT_F);
283 #ifdef __FE_DENORM
284 if (status_value & internal::ExceptionFlags::DENORMAL_F) {
285 raise_helper(internal::ExceptionFlags::DENORMAL_F);
286 }
287 #endif // __FE_DENORM
288
289 // There is no special synchronization scheme available to
290 // raise SEE exceptions. So, we will ignore that for now.
291 // Just plain writing to the MXCSR register does not guarantee
292 // the exception handler will be called.
293
294 return 0;
295 }
296
get_round()297 LIBC_INLINE int get_round() {
298 uint16_t bit_value =
299 (internal::get_mxcsr() >> internal::MXCSR_ROUNDING_CONTROL_BIT_POSITION) &
300 0x3;
301 switch (bit_value) {
302 case internal::RoundingControlValue::TO_NEAREST:
303 return FE_TONEAREST;
304 case internal::RoundingControlValue::DOWNWARD:
305 return FE_DOWNWARD;
306 case internal::RoundingControlValue::UPWARD:
307 return FE_UPWARD;
308 case internal::RoundingControlValue::TOWARD_ZERO:
309 return FE_TOWARDZERO;
310 default:
311 return -1; // Error value.
312 }
313 }
314
set_round(int mode)315 LIBC_INLINE int set_round(int mode) {
316 uint16_t bit_value;
317 switch (mode) {
318 case FE_TONEAREST:
319 bit_value = internal::RoundingControlValue::TO_NEAREST;
320 break;
321 case FE_DOWNWARD:
322 bit_value = internal::RoundingControlValue::DOWNWARD;
323 break;
324 case FE_UPWARD:
325 bit_value = internal::RoundingControlValue::UPWARD;
326 break;
327 case FE_TOWARDZERO:
328 bit_value = internal::RoundingControlValue::TOWARD_ZERO;
329 break;
330 default:
331 return 1; // To indicate failure
332 }
333
334 uint16_t x87_value = static_cast<uint16_t>(
335 bit_value << internal::X87_ROUNDING_CONTROL_BIT_POSITION);
336 uint16_t x87_control = internal::get_x87_control_word();
337 x87_control = static_cast<uint16_t>(
338 (x87_control &
339 ~(uint16_t(0x3) << internal::X87_ROUNDING_CONTROL_BIT_POSITION)) |
340 x87_value);
341 internal::write_x87_control_word(x87_control);
342
343 uint32_t mxcsr_value = bit_value
344 << internal::MXCSR_ROUNDING_CONTROL_BIT_POSITION;
345 uint32_t mxcsr_control = internal::get_mxcsr();
346 mxcsr_control = (mxcsr_control &
347 ~(0x3 << internal::MXCSR_ROUNDING_CONTROL_BIT_POSITION)) |
348 mxcsr_value;
349 internal::write_mxcsr(mxcsr_control);
350
351 return 0;
352 }
353
354 namespace internal {
355
356 #if defined(_WIN32)
357 // MSVC fenv.h defines a very simple representation of the floating point state
358 // which just consists of control and status words of the x87 unit.
359 struct FPState {
360 uint32_t control_word;
361 uint32_t status_word;
362 };
363 #elif defined(__APPLE__)
364 struct FPState {
365 uint16_t control_word;
366 uint16_t status_word;
367 uint32_t mxcsr;
368 uint8_t reserved[8];
369 };
370 #else
371 struct FPState {
372 X87StateDescriptor x87_status;
373 uint32_t mxcsr;
374 };
375 #endif // _WIN32
376
377 } // namespace internal
378
379 static_assert(
380 sizeof(fenv_t) == sizeof(internal::FPState),
381 "Internal floating point state does not match the public fenv_t type.");
382
383 #ifdef _WIN32
384
385 // The exception flags in the Windows FEnv struct and the MXCSR have almost
386 // reversed bit positions.
387 struct WinExceptionFlags {
388 static constexpr uint32_t INEXACT_WIN = 0x01;
389 static constexpr uint32_t UNDERFLOW_WIN = 0x02;
390 static constexpr uint32_t OVERFLOW_WIN = 0x04;
391 static constexpr uint32_t DIV_BY_ZERO_WIN = 0x08;
392 static constexpr uint32_t INVALID_WIN = 0x10;
393 static constexpr uint32_t DENORMAL_WIN = 0x20;
394
395 // The Windows FEnv struct has a second copy of all of these bits in the high
396 // byte of the 32 bit control word. These are used as the source of truth when
397 // calling fesetenv.
398 static constexpr uint32_t HIGH_OFFSET = 24;
399
400 static constexpr uint32_t HIGH_INEXACT = INEXACT_WIN << HIGH_OFFSET;
401 static constexpr uint32_t HIGH_UNDERFLOW = UNDERFLOW_WIN << HIGH_OFFSET;
402 static constexpr uint32_t HIGH_OVERFLOW = OVERFLOW_WIN << HIGH_OFFSET;
403 static constexpr uint32_t HIGH_DIV_BY_ZERO = DIV_BY_ZERO_WIN << HIGH_OFFSET;
404 static constexpr uint32_t HIGH_INVALID = INVALID_WIN << HIGH_OFFSET;
405 static constexpr uint32_t HIGH_DENORMAL = DENORMAL_WIN << HIGH_OFFSET;
406 };
407
408 /*
409 fenv_t control word format:
410
411 Windows (at least for x64) uses a 4 byte control fenv control word stored in
412 a 32 bit integer. The first byte contains just the rounding mode and the
413 exception masks, while the last two bytes contain that same information as
414 well as the flush-to-zero and denormals-are-zero flags. The flags are
415 represented with a truth table:
416
417 00 - No flags set
418 01 - Flush-to-zero and Denormals-are-zero set
419 11 - Flush-to-zero set
420 10 - Denormals-are-zero set
421
422 U represents unused.
423
424 +-----Rounding Mode-----+
425 | |
426 ++ ++
427 || ||
428 RRMMMMMM UUUUUUUU UUUUFFRR UUMMMMMM
429 | | || | |
430 +----+ flags---++ +----+
431 | |
432 +------Exception Masks-----+
433
434
435 fenv_t status word format:
436
437 The status word is a lot simpler for this conversion, since only the
438 exception flags are used in the MXCSR.
439
440 +----+---Exception Flags---+----+
441 | | | |
442 UUEEEEEE UUUUUUUU UUUUUUUU UUEEEEEE
443
444
445
446 MXCSR Format:
447
448 The MXCSR format is the same information, just organized differently. Since
449 the fenv_t struct for windows doesn't include the mxcsr bits, they must be
450 generated from the control word bits.
451
452 Exception Masks---+ +---Exception Flags
453 | |
454 Flush-to-zero---+ +----+ +----+
455 | | | | |
456 FRRMMMMMMDEEEEEE
457 || |
458 ++ +---Denormals-are-zero
459 |
460 +---Rounding Mode
461
462
463 The mask and flag order is as follows:
464
465 fenv_t mxcsr
466
467 denormal inexact
468 invalid underflow
469 div by 0 overflow
470 overflow div by 0
471 underflow denormal
472 inexact invalid
473
474 This is almost reverse, except for denormal and invalid which are in the
475 same order in both.
476 */
477
get_env(fenv_t * envp)478 LIBC_INLINE int get_env(fenv_t *envp) {
479 internal::FPState *state = reinterpret_cast<internal::FPState *>(envp);
480
481 uint32_t status_word = 0;
482 uint32_t control_word = 0;
483
484 uint32_t mxcsr = internal::get_mxcsr();
485
486 // Set exception flags in the status word
487 status_word |= (mxcsr & (internal::ExceptionFlags::INVALID_F |
488 internal::ExceptionFlags::DENORMAL_F))
489 << 4;
490 status_word |= (mxcsr & internal::ExceptionFlags::DIV_BY_ZERO_F) << 1;
491 status_word |= (mxcsr & internal::ExceptionFlags::OVERFLOW_F) >> 1;
492 status_word |= (mxcsr & internal::ExceptionFlags::UNDERFLOW_F) >> 3;
493 status_word |= (mxcsr & internal::ExceptionFlags::INEXACT_F) >> 5;
494 status_word |= status_word << WinExceptionFlags::HIGH_OFFSET;
495
496 // Set exception masks in bits 0-5 and 24-29
497 control_word |= (mxcsr & ((internal::ExceptionFlags::INVALID_F |
498 internal::ExceptionFlags::DENORMAL_F)
499 << 7)) >>
500 3;
501 control_word |= (mxcsr & (internal::ExceptionFlags::DIV_BY_ZERO_F << 7)) >> 6;
502 control_word |= (mxcsr & (internal::ExceptionFlags::OVERFLOW_F << 7)) >> 8;
503 control_word |= (mxcsr & (internal::ExceptionFlags::UNDERFLOW_F << 7)) >> 10;
504 control_word |= (mxcsr & (internal::ExceptionFlags::INEXACT_F << 7)) >> 12;
505 control_word |= control_word << WinExceptionFlags::HIGH_OFFSET;
506
507 // Set rounding in bits 8-9 and 30-31
508 control_word |= (mxcsr & 0x6000) >> 5;
509 control_word |= (mxcsr & 0x6000) << 17;
510
511 // Set flush-to-zero in bit 10
512 control_word |= (mxcsr & 0x8000) >> 5;
513
514 // Set denormals-are-zero xor flush-to-zero in bit 11
515 control_word |= (((mxcsr & 0x8000) >> 9) ^ (mxcsr & 0x0040)) << 5;
516
517 state->control_word = control_word;
518 state->status_word = status_word;
519 return 0;
520 }
521
set_env(const fenv_t * envp)522 LIBC_INLINE int set_env(const fenv_t *envp) {
523 const internal::FPState *state =
524 reinterpret_cast<const internal::FPState *>(envp);
525
526 uint32_t mxcsr = 0;
527
528 // Set exception flags from the status word
529 mxcsr |= static_cast<uint16_t>(
530 (state->status_word &
531 (WinExceptionFlags::HIGH_DENORMAL | WinExceptionFlags::HIGH_INVALID)) >>
532 28);
533 mxcsr |= static_cast<uint16_t>(
534 (state->status_word & WinExceptionFlags::HIGH_DIV_BY_ZERO) >> 25);
535 mxcsr |= static_cast<uint16_t>(
536 (state->status_word & WinExceptionFlags::HIGH_OVERFLOW) >> 23);
537 mxcsr |= static_cast<uint16_t>(
538 (state->status_word & WinExceptionFlags::HIGH_UNDERFLOW) >> 21);
539 mxcsr |= static_cast<uint16_t>(
540 (state->status_word & WinExceptionFlags::HIGH_INEXACT) >> 19);
541
542 // Set denormals-are-zero from bit 10 xor bit 11
543 mxcsr |= static_cast<uint16_t>(
544 (((state->control_word & 0x800) >> 1) ^ (state->control_word & 0x400)) >>
545 4);
546
547 // Set exception masks from bits 24-29
548 mxcsr |= static_cast<uint16_t>(
549 (state->control_word &
550 (WinExceptionFlags::HIGH_DENORMAL | WinExceptionFlags::HIGH_INVALID)) >>
551 21);
552 mxcsr |= static_cast<uint16_t>(
553 (state->control_word & WinExceptionFlags::HIGH_DIV_BY_ZERO) >> 18);
554 mxcsr |= static_cast<uint16_t>(
555 (state->control_word & WinExceptionFlags::HIGH_OVERFLOW) >> 16);
556 mxcsr |= static_cast<uint16_t>(
557 (state->control_word & WinExceptionFlags::HIGH_UNDERFLOW) >> 14);
558 mxcsr |= static_cast<uint16_t>(
559 (state->control_word & WinExceptionFlags::HIGH_INEXACT) >> 12);
560
561 // Set rounding from bits 30-31
562 mxcsr |= static_cast<uint16_t>((state->control_word & 0xc0000000) >> 17);
563
564 // Set flush-to-zero from bit 10
565 mxcsr |= static_cast<uint16_t>((state->control_word & 0x400) << 5);
566
567 internal::write_mxcsr(mxcsr);
568 return 0;
569 }
570 #else
get_env(fenv_t * envp)571 LIBC_INLINE int get_env(fenv_t *envp) {
572 internal::FPState *state = reinterpret_cast<internal::FPState *>(envp);
573 #ifdef __APPLE__
574 internal::X87StateDescriptor x87_status;
575 internal::get_x87_state_descriptor(x87_status);
576 state->control_word = x87_status.control_word;
577 state->status_word = x87_status.status_word;
578 #else
579 internal::get_x87_state_descriptor(state->x87_status);
580 #endif // __APPLE__
581 state->mxcsr = internal::get_mxcsr();
582 return 0;
583 }
584
set_env(const fenv_t * envp)585 LIBC_INLINE int set_env(const fenv_t *envp) {
586 // envp contains everything including pieces like the current
587 // top of FPU stack. We cannot arbitrarily change them. So, we first
588 // read the current status and update only those pieces which are
589 // not disruptive.
590 internal::X87StateDescriptor x87_status;
591 internal::get_x87_state_descriptor(x87_status);
592
593 if (envp == FE_DFL_ENV) {
594 // Reset the exception flags in the status word.
595 x87_status.status_word &= ~uint16_t(0x3F);
596 // Reset other non-sensitive parts of the status word.
597 for (int i = 0; i < 5; i++)
598 x87_status._[i] = 0;
599 // In the control word, we do the following:
600 // 1. Mask all exceptions
601 // 2. Set rounding mode to round-to-nearest
602 // 3. Set the internal precision to double extended precision.
603 x87_status.control_word |= uint16_t(0x3F); // Mask all exceptions.
604 x87_status.control_word &= ~(uint16_t(0x3) << 10); // Round to nearest.
605 x87_status.control_word |= (uint16_t(0x3) << 8); // Extended precision.
606 internal::write_x87_state_descriptor(x87_status);
607
608 // We take the exact same approach MXCSR register as well.
609 // MXCSR has two additional fields, "flush-to-zero" and
610 // "denormals-are-zero". We reset those bits. Also, MXCSR does not
611 // have a field which controls the precision of internal operations.
612 uint32_t mxcsr = internal::get_mxcsr();
613 mxcsr &= ~uint16_t(0x3F); // Clear exception flags.
614 mxcsr &= ~(uint16_t(0x1) << 6); // Reset denormals-are-zero
615 mxcsr |= (uint16_t(0x3F) << 7); // Mask exceptions
616 mxcsr &= ~(uint16_t(0x3) << 13); // Round to nearest.
617 mxcsr &= ~(uint16_t(0x1) << 15); // Reset flush-to-zero
618 internal::write_mxcsr(mxcsr);
619
620 return 0;
621 }
622
623 const internal::FPState *fpstate =
624 reinterpret_cast<const internal::FPState *>(envp);
625
626 // Copy the exception status flags from envp.
627 x87_status.status_word &= ~uint16_t(0x3F);
628 #ifdef __APPLE__
629 x87_status.status_word |= (fpstate->status_word & 0x3F);
630 // We can set the x87 control word as is as there no sensitive bits.
631 x87_status.control_word = fpstate->control_word;
632 #else
633 x87_status.status_word |= (fpstate->x87_status.status_word & 0x3F);
634 // Copy other non-sensitive parts of the status word.
635 for (int i = 0; i < 5; i++)
636 x87_status._[i] = fpstate->x87_status._[i];
637 // We can set the x87 control word as is as there no sensitive bits.
638 x87_status.control_word = fpstate->x87_status.control_word;
639 #endif // __APPLE__
640 internal::write_x87_state_descriptor(x87_status);
641
642 // We can write the MXCSR state as is as there are no sensitive bits.
643 internal::write_mxcsr(fpstate->mxcsr);
644 return 0;
645 }
646 #endif
647
648 } // namespace fputil
649 } // namespace LIBC_NAMESPACE_DECL
650
651 #endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FENVIMPL_H
652