52  * The "accumulator" word is nominally a 130-bit value. We split it into
53  * words of 44 bits, each held in a 64-bit variable.
64  * We want to reduce that value modulo p = 2^130-5, so W^3 = 20 mod p,
80  * bits of u1. Note that since r is clamped down to a 124-bit value, the
95  * need to compute the additions (for the bx values) over 128-bit
96  * quantities; we can stick to 64-bit computations.
99  * Since the 128-bit result of a 64x64 multiplication is actually
100  * represented over two 64-bit registers, it is cheaper to arrange for
102  * that 64-bit boundary. This is done by left shifting the rx, ux and tx
129 	r1 = r[1];  in poly1305_inner_big()
135 	a1 = acc[1];  in poly1305_inner_big()
140 		uint64_t v0, v1, v2;  in poly1305_inner_big()  local
143 		v0 = br_dec64le(buf + 0);  in poly1305_inner_big()
146 		v1 = ((v0 >> 44) | (v1 << 20)) & MASK44;  in poly1305_inner_big()
147 		v0 &= MASK44;  in poly1305_inner_big()
148 		a0 += v0;  in poly1305_inner_big()
150 		a2 += v2 + ((uint64_t)1 << 40);  in poly1305_inner_big()
158 		v0 = br_dec64le(buf + 16);  in poly1305_inner_big()
161 		v1 = ((v0 >> 44) | (v1 << 20)) & MASK44;  in poly1305_inner_big()
162 		v0 &= MASK44;  in poly1305_inner_big()
163 		a0 += v0;  in poly1305_inner_big()
165 		a2 += v2 + ((uint64_t)1 << 40);  in poly1305_inner_big()
173 		v0 = br_dec64le(buf + 32);  in poly1305_inner_big()
176 		v1 = ((v0 >> 44) | (v1 << 20)) & MASK44;  in poly1305_inner_big()
177 		v0 &= MASK44;  in poly1305_inner_big()
178 		a0 += v0;  in poly1305_inner_big()
180 		a2 += v2 + ((uint64_t)1 << 40);  in poly1305_inner_big()
188 		v0 = br_dec64le(buf + 48);  in poly1305_inner_big()
191 		v1 = ((v0 >> 44) | (v1 << 20)) & MASK44;  in poly1305_inner_big()
192 		v0 &= MASK44;  in poly1305_inner_big()
193 		a0 += v0;  in poly1305_inner_big()
195 		a2 += v2 + ((uint64_t)1 << 40);  in poly1305_inner_big()
211 		len -= 64;  in poly1305_inner_big()
214 	acc[1] = a1;  in poly1305_inner_big()
228 	r1 = r[1];  in poly1305_inner_small()
234 	a1 = acc[1];  in poly1305_inner_small()
239 		uint64_t v0, v1, v2;  in poly1305_inner_small()  local
245 			memset(tmp + len, 0, (sizeof tmp) - len);  in poly1305_inner_small()
249 		v0 = br_dec64le(buf + 0);  in poly1305_inner_small()
253 		v1 = ((v0 >> 44) | (v1 << 20)) & MASK44;  in poly1305_inner_small()
254 		v0 &= MASK44;  in poly1305_inner_small()
256 		a0 += v0;  in poly1305_inner_small()
258 		a2 += v2 + ((uint64_t)1 << 40);  in poly1305_inner_small()
291 		len -= 16;  in poly1305_inner_small()
294 	acc[1] = a1;  in poly1305_inner_small()
307 		len -= len2;  in poly1305_inner()
322 	uint32_t v0, v1, v2, v3, v4;  in br_poly1305_ctmulq_run()  local
338 		ichacha(key, iv, 1, data, len);  in br_poly1305_ctmulq_run()
360 	 * Decode the 'r' value into 44-bit words, left-shifted by 20 bits.  in br_poly1305_ctmulq_run()
366 	r[1] = ((r0 >> 24) | (r1 << 40)) & ~(uint64_t)0xFFFFF;  in br_poly1305_ctmulq_run()
368 	r1 = 20 * (r[1] >> 20);  in br_poly1305_ctmulq_run()
377 	acc[1] = 0;  in br_poly1305_ctmulq_run()
392 	 * in three 44-bit values (the lowest one might be slightly above  in br_poly1305_ctmulq_run()
395 	acc[1] += (acc[0] >> 44);  in br_poly1305_ctmulq_run()
397 	acc[2] += (acc[1] >> 44);  in br_poly1305_ctmulq_run()
398 	acc[1] &= MASK44;  in br_poly1305_ctmulq_run()
401 	acc[1] += (acc[0] >> 44);  in br_poly1305_ctmulq_run()
403 	acc[2] += (acc[1] >> 44);  in br_poly1305_ctmulq_run()
404 	acc[1] &= MASK44;  in br_poly1305_ctmulq_run()
409 	 * The value may still fall in the 2^130-5..2^130-1 range, in  in br_poly1305_ctmulq_run()
411 	 * in constant-time, between 'acc' and 'acc-p'. We encode the  in br_poly1305_ctmulq_run()
412 	 * value over four 32-bit integers to finish the operation.  in br_poly1305_ctmulq_run()
414 	v0 = (uint32_t)acc[0];  in br_poly1305_ctmulq_run()
415 	v1 = (uint32_t)(acc[0] >> 32) | ((uint32_t)acc[1] << 12);  in br_poly1305_ctmulq_run()
416 	v2 = (uint32_t)(acc[1] >> 20) | ((uint32_t)acc[2] << 24);  in br_poly1305_ctmulq_run()
420 	ctl = GT(v0, 0xFFFFFFFA);  in br_poly1305_ctmulq_run()
425 	v0 = MUX(ctl, v0 + 5, v0);  in br_poly1305_ctmulq_run()
434 	w0 = (uint64_t)v0 + (uint64_t)br_dec32le(pkey + 16);  in br_poly1305_ctmulq_run()
438 	v0 = (uint32_t)w0;  in br_poly1305_ctmulq_run()
446 	br_enc32le((unsigned char *)tag +  0, v0);  in br_poly1305_ctmulq_run()
455 		ichacha(key, iv, 1, data, len);  in br_poly1305_ctmulq_run()