1 /*
2 * Double-precision vector e^x function.
3 *
4 * Copyright (c) 2019-2024, Arm Limited.
5 * SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
6 */
7
8 #include "mathlib.h"
9 #include "v_math.h"
10 #include "test_defs.h"
11 #include "test_sig.h"
12
13 #define N (1 << V_EXP_TABLE_BITS)
14 #define IndexMask (N - 1)
15
16 const static volatile struct
17 {
18 float64x2_t poly[3];
19 float64x2_t inv_ln2, ln2_hi, ln2_lo, shift;
20 #if !WANT_SIMD_EXCEPT
21 float64x2_t special_bound, scale_thresh;
22 #endif
23 } data = {
24 /* maxerr: 1.88 +0.5 ulp
25 rel error: 1.4337*2^-53
26 abs error: 1.4299*2^-53 in [ -ln2/256, ln2/256 ]. */
27 .poly = { V2 (0x1.ffffffffffd43p-2), V2 (0x1.55555c75adbb2p-3),
28 V2 (0x1.55555da646206p-5) },
29 #if !WANT_SIMD_EXCEPT
30 .scale_thresh = V2 (163840.0), /* 1280.0 * N. */
31 .special_bound = V2 (704.0),
32 #endif
33 .inv_ln2 = V2 (0x1.71547652b82fep7), /* N/ln2. */
34 .ln2_hi = V2 (0x1.62e42fefa39efp-8), /* ln2/N. */
35 .ln2_lo = V2 (0x1.abc9e3b39803f3p-63),
36 .shift = V2 (0x1.8p+52)
37 };
38
39 #define C(i) data.poly[i]
40 #define Tab __v_exp_data
41
42 #if WANT_SIMD_EXCEPT
43
44 # define TinyBound v_u64 (0x2000000000000000) /* asuint64 (0x1p-511). */
45 # define BigBound v_u64 (0x4080000000000000) /* asuint64 (0x1p9). */
46 # define SpecialBound v_u64 (0x2080000000000000) /* BigBound - TinyBound. */
47
48 static float64x2_t VPCS_ATTR NOINLINE
special_case(float64x2_t x,float64x2_t y,uint64x2_t cmp)49 special_case (float64x2_t x, float64x2_t y, uint64x2_t cmp)
50 {
51 /* If fenv exceptions are to be triggered correctly, fall back to the scalar
52 routine to special lanes. */
53 return v_call_f64 (exp, x, y, cmp);
54 }
55
56 #else
57
58 # define SpecialOffset v_u64 (0x6000000000000000) /* 0x1p513. */
59 /* SpecialBias1 + SpecialBias1 = asuint(1.0). */
60 # define SpecialBias1 v_u64 (0x7000000000000000) /* 0x1p769. */
61 # define SpecialBias2 v_u64 (0x3010000000000000) /* 0x1p-254. */
62
63 static inline float64x2_t VPCS_ATTR
special_case(float64x2_t s,float64x2_t y,float64x2_t n)64 special_case (float64x2_t s, float64x2_t y, float64x2_t n)
65 {
66 /* 2^(n/N) may overflow, break it up into s1*s2. */
67 uint64x2_t b = vandq_u64 (vcltzq_f64 (n), SpecialOffset);
68 float64x2_t s1 = vreinterpretq_f64_u64 (vsubq_u64 (SpecialBias1, b));
69 float64x2_t s2 = vreinterpretq_f64_u64 (
70 vaddq_u64 (vsubq_u64 (vreinterpretq_u64_f64 (s), SpecialBias2), b));
71 uint64x2_t cmp = vcagtq_f64 (n, data.scale_thresh);
72 float64x2_t r1 = vmulq_f64 (s1, s1);
73 float64x2_t r0 = vmulq_f64 (vfmaq_f64 (s2, y, s2), s1);
74 return vbslq_f64 (cmp, r1, r0);
75 }
76
77 #endif
78
V_NAME_D1(exp)79 float64x2_t VPCS_ATTR V_NAME_D1 (exp) (float64x2_t x)
80 {
81 float64x2_t n, r, r2, s, y, z;
82 uint64x2_t cmp, u, e;
83
84 #if WANT_SIMD_EXCEPT
85 /* If any lanes are special, mask them with 1 and retain a copy of x to allow
86 special_case to fix special lanes later. This is only necessary if fenv
87 exceptions are to be triggered correctly. */
88 float64x2_t xm = x;
89 uint64x2_t iax = vreinterpretq_u64_f64 (vabsq_f64 (x));
90 cmp = vcgeq_u64 (vsubq_u64 (iax, TinyBound), SpecialBound);
91 if (unlikely (v_any_u64 (cmp)))
92 x = vbslq_f64 (cmp, v_f64 (1), x);
93 #else
94 cmp = vcagtq_f64 (x, data.special_bound);
95 #endif
96
97 /* n = round(x/(ln2/N)). */
98 z = vfmaq_f64 (data.shift, x, data.inv_ln2);
99 u = vreinterpretq_u64_f64 (z);
100 n = vsubq_f64 (z, data.shift);
101
102 /* r = x - n*ln2/N. */
103 r = x;
104 r = vfmsq_f64 (r, data.ln2_hi, n);
105 r = vfmsq_f64 (r, data.ln2_lo, n);
106
107 e = vshlq_n_u64 (u, 52 - V_EXP_TABLE_BITS);
108
109 /* y = exp(r) - 1 ~= r + C0 r^2 + C1 r^3 + C2 r^4. */
110 r2 = vmulq_f64 (r, r);
111 y = vfmaq_f64 (C (0), C (1), r);
112 y = vfmaq_f64 (y, C (2), r2);
113 y = vfmaq_f64 (r, y, r2);
114
115 /* s = 2^(n/N). */
116 u = (uint64x2_t){ Tab[u[0] & IndexMask], Tab[u[1] & IndexMask] };
117 s = vreinterpretq_f64_u64 (vaddq_u64 (u, e));
118
119 if (unlikely (v_any_u64 (cmp)))
120 #if WANT_SIMD_EXCEPT
121 return special_case (xm, vfmaq_f64 (s, y, s), cmp);
122 #else
123 return special_case (s, y, n);
124 #endif
125
126 return vfmaq_f64 (s, y, s);
127 }
128
129 TEST_SIG (V, D, 1, exp, -9.9, 9.9)
130 TEST_ULP (V_NAME_D1 (exp), 1.9)
131 TEST_DISABLE_FENV_IF_NOT (V_NAME_D1 (exp), WANT_SIMD_EXCEPT)
132 TEST_INTERVAL (V_NAME_D1 (exp), 0, 0xffff000000000000, 10000)
133 TEST_SYM_INTERVAL (V_NAME_D1 (exp), 0x1p-6, 0x1p6, 400000)
134 TEST_SYM_INTERVAL (V_NAME_D1 (exp), 633.3, 733.3, 10000)
135