Home
last modified time | relevance | path

Searched refs:precision (Results 1 – 25 of 94) sorted by relevance

1234

/linux/drivers/firmware/efi/libstub/
H A Dvsprintf.c315 int precision; /* min. # of digits for integers; max in vsnprintf() local
357 precision = -1; in vsnprintf()
360 precision = get_int(&fmt, &args); in vsnprintf()
361 if (precision >= 0) in vsnprintf()
385 precision = INT_MAX; in vsnprintf()
389 precision = len = 1; in vsnprintf()
395 if (precision < 0) in vsnprintf()
396 precision = INT_MAX; in vsnprintf()
399 s = precision < 6 ? "" : "(null)"; in vsnprintf()
403 precision = len = utf16s_utf8nlen((const u16 *)s, precision); in vsnprintf()
[all …]
/linux/arch/alpha/boot/
H A Dstdio.c43 static char * number(char * str, unsigned long long num, int base, int size, int precision, int typ… in number() argument
82 if (i > precision) in number()
83 precision = i; in number()
84 size -= precision; in number()
101 while (i < precision--) in number()
121 int precision; /* min. # of digits for integers; max in vsprintf() local
161 precision = -1; in vsprintf()
165 precision = skip_atoi(&fmt); in vsprintf()
169 precision = va_arg(args, int); in vsprintf()
171 if (precision < 0) in vsprintf()
[all …]
/linux/tools/testing/selftests/kvm/lib/
H A Dguest_sprintf.c44 int precision, int type) in number() argument
89 if (i > precision) in number()
90 precision = i; in number()
91 size -= precision; in number()
108 while (i < precision--) in number()
129 int precision; /* in guest_vsnprintf() local
182 precision = -1; in guest_vsnprintf()
186 precision = skip_atoi(&fmt); in guest_vsnprintf()
190 precision = va_arg(args, int); in guest_vsnprintf()
192 if (precision < 0) in guest_vsnprintf()
[all …]
/linux/arch/arm/nwfpe/
H A Dsoftfloat-specialize71 The pattern for a default generated single-precision NaN.
78 Returns 1 if the single-precision floating-point value `a' is a NaN;
91 Returns 1 if the single-precision floating-point value `a' is a signaling
104 Returns the result of converting the single-precision floating-point NaN
124 precision floating-point format.
136 Takes two single-precision floating-point values `a' and `b', one of which
163 The pattern for a default generated double-precision NaN.
170 Returns 1 if the double-precision floating-point value `a' is a NaN;
183 Returns 1 if the double-precision floating-point value `a' is a signaling
198 Returns the result of converting the double-precision floating-point NaN
[all …]
/linux/arch/m68k/fpsp040/
H A Dutil.S96 | If the instruction is fsgldiv of fsglmul, the rounding precision must be
98 | precision instruction, the rounding precision is then set to the force
99 | precision.
109 beql ovff_sgl |force precision is single
111 beql ovff_dbl |force precision is double
136 | Inst is either fsgldiv or fsglmul. Force extended precision.
149 | The precision is in the fpcr.
152 bfextu FPCR_MODE(%a6){#0:#2},%d0 |set round precision
164 | This sets the round precision according to the destination size.
170 | ;is the rounding precision
[all …]
H A Dx_store.S100 | dest_dbl --- write double precision value to user space
104 | a1 -> source in extended precision
110 |Changes extended precision to double precision.
133 subw #0x3fff,%d0 |subtract extended precision bias
136 addw #0x3ff,%d0 |add double precision bias
166 movel #0x8,%d0 |byte count for double precision number
H A Dslog2.S19 | to double precision. The result is provably monotonic
20 | in double precision.
35 | traps, and precision control = double extended.
50 | traps, and precision control = double extended.
64 | traps, and precision control = double extended.
79 | traps, and precision control = double extended.
H A Dstwotox.S16 | result is subsequently rounded to double precision. The
17 | result is provably monotonic in double precision.
199 fmovel %d1,%fpcr | ...set user's rounding mode/precision
303 fmovel %d1,%fpcr | ...set user's rounding mode/precision
H A Dres_func.S112 | destination precision and mode.
476 | the round precision is found in the fpcr. If the rounding precision
486 | The move or operation is not with forced precision. Use the
496 | The move is fdmove or round precision is double. Result is zero.
511 | The move is fsmove or round precision is single. Result is zero.
526 | The precision is extended, so the result in etemp is correct.
636 | call round with user's precision and mode
646 | call round with user's precision and mode
766 btstl #6,%d0 |test for forced precision
787 | ;round precision/mode. This
[all …]
H A Dsacos.S15 | result is subsequently rounded to double precision. The
16 | result is provably monotonic in double precision.
H A Dsetox.S22 | to double precision. The result is provably monotonic in double
23 | precision.
93 | constant := single-precision( 64/log 2 ).
95 | Using a single-precision constant avoids memory access.
96 | Another effect of using a single-precision "constant" is
104 | 3.1 R := X + N*L1, where L1 := single-precision(-log2/64).
105 | 3.2 R := R + N*L2, L2 := extended-precision(-log2/64 - L1).
131 | are single precision; A2 and A3 are double precision.
145 | 2^(J/64) to roughly 85 bits; T is in extended precision
146 | and t is in single precision. Note also that T is rounded
H A Dround.S21 | round --- round result according to precision/mode
24 | d1(high word) contains rounding precision:
50 | :rounding precision and sets
130 | selected rounding precision. It is called by the round subroutine
138 swap %d1 |have d1.w point to round precision
406 | d0: rounding precision
413 | d0 comes into this routine with the rounding precision. It
415 | rounding precision.
425 cmpib #0,%d0 |if 0 then extended precision
436 cmpil #1,%d0 |if 1 then single precision
H A Dsgetem.S7 | precision number in fp0. sGETEXPD handles denormalized
12 | extended precision number and returned in fp0. The
79 andil #0xffffff00,%d0 |clear rounding precision and mode
H A Dsasin.S15 | result is subsequently rounded to double precision. The
16 | result is provably monotonic in double precision.
H A Dsatanh.S16 | result is subsequently rounded to double precision. The
17 | result is provably monotonic in double precision.
H A Dbindec.S7 | Converts an input in extended precision format
11 | a0 points to the input extended precision value
33 | value is viewed as 2^^e * 1.f in extended precision.
143 | Constants in extended precision
147 | Constants in single precision
219 | as 2^^e * 1.f in extended precision. This value is stored
553 | mode and precision. The original FPCR is saved in L_SCR1.
613 | in extended precision, so the use of a previous power-of-ten
/linux/include/linux/
H A Daverage.h58 unsigned long precision = _precision; \
67 (val << precision)) >> weight_rcp : \
68 (val << precision)); \
/linux/Documentation/arch/arm/nwfpe/
H A Dnwfpe.rst28 Arithmetic. As many as four formats are supported: single precision,
29 double precision, extended double precision, and quadruple precision.
31 conversions to and from decimal. We use only the single precision,
32 double precision and extended double precision formats. The port of
H A Dnotes.rst23 extended, then does the multiply in extended precision.
32 in extended precision, due to the stfe instruction used to save f4 in log(y).
/linux/tools/testing/selftests/timers/
H A Dfreq-step.c40 static double precision; variable
224 precision = get_sample(&sample) / 2.0; in init_test()
226 1e9 * precision); in init_test()
228 if (precision > MAX_PRECISION) in init_test()
230 1e9 * precision, 1e9 * MAX_PRECISION); in init_test()
/linux/drivers/accel/ethosu/
H A Dethosu_gem.c130 u16 precision; member
197 u32 storage = fm->precision >> 14; in feat_matrix_chained()
207 u32 element_size, storage = ethosu_is_u65(edev) ? 0 : fm->precision >> 14; in feat_matrix_length()
245 switch ((fm->precision >> 6) & 0x3) { // format in feat_matrix_length()
247 element_size = BIT((fm->precision >> (ofm ? 1 : 2)) & 0x3); in feat_matrix_length()
251 element_size = BIT((fm->precision >> (ofm ? 1 : 2)) & 0x3); in feat_matrix_length()
450 use_scale = !(st.ofm.precision & 0x100); in ethosu_gem_cmdstream_copy_and_validate()
458 use_scale = !(st.ofm.precision & 0x100); in ethosu_gem_cmdstream_copy_and_validate()
502 st.ifm.precision = param; in ethosu_gem_cmdstream_copy_and_validate()
545 st.ofm.precision = param; in ethosu_gem_cmdstream_copy_and_validate()
[all …]
/linux/tools/testing/selftests/timens/
H A Dtimens.c92 double precision = 0.0; in test_gettime() local
100 precision = -2.0; in test_gettime()
119 if (difftime(cur_ts.tv_sec, child_ts_new.tv_sec) < precision) { in test_gettime()
/linux/include/uapi/linux/
H A Dtimex.h73 __kernel_long_t precision;/* clock precision (usec) (read only) */ member
112 long long precision;/* clock precision (usec) (read only) */ member
/linux/arch/m68k/ifpsp060/
H A Dfplsp.doc79 There are 3 entry-points for each instruction type: single precision,
80 double precision, and extended precision.
83 extended precision operand if program executes:
104 are passed in single precision format.
/linux/arch/m68k/ifpsp060/src/
H A Dfpsp.S1427 # or double precision denorm, inf, or nan, the operand needs to be
1428 # "corrected" in order to have the proper equivalent extended precision
1587 # the extended precision result is still in fp0. but, we need to save it
1722 # underflow can happen for extended precision. extended precision opclass
2403 # (1) FP Instructions using extended precision or packed immediate #
2966 # instruction is using an extended precision immediate operand. Therefore,
3293 # For the case of an extended precision opclass 3 instruction, #
5309 # here, the operation may underflow iff the precision is sgl or dbl.
6147 # a0 = pointer to extended precision input #
6148 # d0 = round precision,mode #
[all …]

1234