1*0957b409SSimon J. Gerraty /*
2*0957b409SSimon J. Gerraty * Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
3*0957b409SSimon J. Gerraty *
4*0957b409SSimon J. Gerraty * Permission is hereby granted, free of charge, to any person obtaining
5*0957b409SSimon J. Gerraty * a copy of this software and associated documentation files (the
6*0957b409SSimon J. Gerraty * "Software"), to deal in the Software without restriction, including
7*0957b409SSimon J. Gerraty * without limitation the rights to use, copy, modify, merge, publish,
8*0957b409SSimon J. Gerraty * distribute, sublicense, and/or sell copies of the Software, and to
9*0957b409SSimon J. Gerraty * permit persons to whom the Software is furnished to do so, subject to
10*0957b409SSimon J. Gerraty * the following conditions:
11*0957b409SSimon J. Gerraty *
12*0957b409SSimon J. Gerraty * The above copyright notice and this permission notice shall be
13*0957b409SSimon J. Gerraty * included in all copies or substantial portions of the Software.
14*0957b409SSimon J. Gerraty *
15*0957b409SSimon J. Gerraty * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16*0957b409SSimon J. Gerraty * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17*0957b409SSimon J. Gerraty * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18*0957b409SSimon J. Gerraty * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
19*0957b409SSimon J. Gerraty * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
20*0957b409SSimon J. Gerraty * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
21*0957b409SSimon J. Gerraty * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22*0957b409SSimon J. Gerraty * SOFTWARE.
23*0957b409SSimon J. Gerraty */
24*0957b409SSimon J. Gerraty
25*0957b409SSimon J. Gerraty #include "inner.h"
26*0957b409SSimon J. Gerraty
27*0957b409SSimon J. Gerraty /* see inner.h */
28*0957b409SSimon J. Gerraty void
br_i15_montymul(uint16_t * d,const uint16_t * x,const uint16_t * y,const uint16_t * m,uint16_t m0i)29*0957b409SSimon J. Gerraty br_i15_montymul(uint16_t *d, const uint16_t *x, const uint16_t *y,
30*0957b409SSimon J. Gerraty const uint16_t *m, uint16_t m0i)
31*0957b409SSimon J. Gerraty {
32*0957b409SSimon J. Gerraty size_t len, len4, u, v;
33*0957b409SSimon J. Gerraty uint32_t dh;
34*0957b409SSimon J. Gerraty
35*0957b409SSimon J. Gerraty len = (m[0] + 15) >> 4;
36*0957b409SSimon J. Gerraty len4 = len & ~(size_t)3;
37*0957b409SSimon J. Gerraty br_i15_zero(d, m[0]);
38*0957b409SSimon J. Gerraty dh = 0;
39*0957b409SSimon J. Gerraty for (u = 0; u < len; u ++) {
40*0957b409SSimon J. Gerraty uint32_t f, xu, r, zh;
41*0957b409SSimon J. Gerraty
42*0957b409SSimon J. Gerraty xu = x[u + 1];
43*0957b409SSimon J. Gerraty f = MUL15((d[1] + MUL15(x[u + 1], y[1])) & 0x7FFF, m0i)
44*0957b409SSimon J. Gerraty & 0x7FFF;
45*0957b409SSimon J. Gerraty #if BR_ARMEL_CORTEXM_GCC
46*0957b409SSimon J. Gerraty if (len4 != 0) {
47*0957b409SSimon J. Gerraty uint16_t *limit;
48*0957b409SSimon J. Gerraty
49*0957b409SSimon J. Gerraty limit = d + len4;
50*0957b409SSimon J. Gerraty asm volatile (
51*0957b409SSimon J. Gerraty "\n\
52*0957b409SSimon J. Gerraty @ carry: r=r2 \n\
53*0957b409SSimon J. Gerraty @ multipliers: xu=r3 f=r4 \n\
54*0957b409SSimon J. Gerraty @ base registers: d+v=r5 y+v=r6 m+v=r7 \n\
55*0957b409SSimon J. Gerraty @ r8 contains 0x7FFF \n\
56*0957b409SSimon J. Gerraty @ r9 contains d+len4 \n\
57*0957b409SSimon J. Gerraty ldr r0, %[limit] \n\
58*0957b409SSimon J. Gerraty ldr r3, %[xu] \n\
59*0957b409SSimon J. Gerraty mov r9, r0 \n\
60*0957b409SSimon J. Gerraty ldr r4, %[f] \n\
61*0957b409SSimon J. Gerraty eor r2, r2 \n\
62*0957b409SSimon J. Gerraty ldr r5, %[d] \n\
63*0957b409SSimon J. Gerraty sub r1, r2, #1 \n\
64*0957b409SSimon J. Gerraty ldr r6, %[y] \n\
65*0957b409SSimon J. Gerraty lsr r1, r1, #17 \n\
66*0957b409SSimon J. Gerraty ldr r7, %[m] \n\
67*0957b409SSimon J. Gerraty mov r8, r1 \n\
68*0957b409SSimon J. Gerraty loop%=: \n\
69*0957b409SSimon J. Gerraty ldrh r0, [r6, #2] \n\
70*0957b409SSimon J. Gerraty ldrh r1, [r7, #2] \n\
71*0957b409SSimon J. Gerraty mul r0, r3 \n\
72*0957b409SSimon J. Gerraty mul r1, r4 \n\
73*0957b409SSimon J. Gerraty add r2, r0, r2 \n\
74*0957b409SSimon J. Gerraty ldrh r0, [r5, #2] \n\
75*0957b409SSimon J. Gerraty add r2, r1, r2 \n\
76*0957b409SSimon J. Gerraty mov r1, r8 \n\
77*0957b409SSimon J. Gerraty add r2, r0, r2 \n\
78*0957b409SSimon J. Gerraty and r1, r2 \n\
79*0957b409SSimon J. Gerraty lsr r2, r2, #15 \n\
80*0957b409SSimon J. Gerraty strh r1, [r5, #0] \n\
81*0957b409SSimon J. Gerraty \n\
82*0957b409SSimon J. Gerraty ldrh r0, [r6, #4] \n\
83*0957b409SSimon J. Gerraty ldrh r1, [r7, #4] \n\
84*0957b409SSimon J. Gerraty mul r0, r3 \n\
85*0957b409SSimon J. Gerraty mul r1, r4 \n\
86*0957b409SSimon J. Gerraty add r2, r0, r2 \n\
87*0957b409SSimon J. Gerraty ldrh r0, [r5, #4] \n\
88*0957b409SSimon J. Gerraty add r2, r1, r2 \n\
89*0957b409SSimon J. Gerraty mov r1, r8 \n\
90*0957b409SSimon J. Gerraty add r2, r0, r2 \n\
91*0957b409SSimon J. Gerraty and r1, r2 \n\
92*0957b409SSimon J. Gerraty lsr r2, r2, #15 \n\
93*0957b409SSimon J. Gerraty strh r1, [r5, #2] \n\
94*0957b409SSimon J. Gerraty \n\
95*0957b409SSimon J. Gerraty ldrh r0, [r6, #6] \n\
96*0957b409SSimon J. Gerraty ldrh r1, [r7, #6] \n\
97*0957b409SSimon J. Gerraty mul r0, r3 \n\
98*0957b409SSimon J. Gerraty mul r1, r4 \n\
99*0957b409SSimon J. Gerraty add r2, r0, r2 \n\
100*0957b409SSimon J. Gerraty ldrh r0, [r5, #6] \n\
101*0957b409SSimon J. Gerraty add r2, r1, r2 \n\
102*0957b409SSimon J. Gerraty mov r1, r8 \n\
103*0957b409SSimon J. Gerraty add r2, r0, r2 \n\
104*0957b409SSimon J. Gerraty and r1, r2 \n\
105*0957b409SSimon J. Gerraty lsr r2, r2, #15 \n\
106*0957b409SSimon J. Gerraty strh r1, [r5, #4] \n\
107*0957b409SSimon J. Gerraty \n\
108*0957b409SSimon J. Gerraty ldrh r0, [r6, #8] \n\
109*0957b409SSimon J. Gerraty ldrh r1, [r7, #8] \n\
110*0957b409SSimon J. Gerraty mul r0, r3 \n\
111*0957b409SSimon J. Gerraty mul r1, r4 \n\
112*0957b409SSimon J. Gerraty add r2, r0, r2 \n\
113*0957b409SSimon J. Gerraty ldrh r0, [r5, #8] \n\
114*0957b409SSimon J. Gerraty add r2, r1, r2 \n\
115*0957b409SSimon J. Gerraty mov r1, r8 \n\
116*0957b409SSimon J. Gerraty add r2, r0, r2 \n\
117*0957b409SSimon J. Gerraty and r1, r2 \n\
118*0957b409SSimon J. Gerraty lsr r2, r2, #15 \n\
119*0957b409SSimon J. Gerraty strh r1, [r5, #6] \n\
120*0957b409SSimon J. Gerraty \n\
121*0957b409SSimon J. Gerraty add r5, r5, #8 \n\
122*0957b409SSimon J. Gerraty add r6, r6, #8 \n\
123*0957b409SSimon J. Gerraty add r7, r7, #8 \n\
124*0957b409SSimon J. Gerraty cmp r5, r9 \n\
125*0957b409SSimon J. Gerraty bne loop%= \n\
126*0957b409SSimon J. Gerraty \n\
127*0957b409SSimon J. Gerraty str r2, %[carry] \n\
128*0957b409SSimon J. Gerraty "
129*0957b409SSimon J. Gerraty : [carry] "=m" (r)
130*0957b409SSimon J. Gerraty : [xu] "m" (xu), [f] "m" (f), [d] "m" (d), [y] "m" (y),
131*0957b409SSimon J. Gerraty [m] "m" (m), [limit] "m" (limit)
132*0957b409SSimon J. Gerraty : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9" );
133*0957b409SSimon J. Gerraty } else {
134*0957b409SSimon J. Gerraty r = 0;
135*0957b409SSimon J. Gerraty }
136*0957b409SSimon J. Gerraty v = len4;
137*0957b409SSimon J. Gerraty #else
138*0957b409SSimon J. Gerraty r = 0;
139*0957b409SSimon J. Gerraty for (v = 0; v < len4; v += 4) {
140*0957b409SSimon J. Gerraty uint32_t z;
141*0957b409SSimon J. Gerraty
142*0957b409SSimon J. Gerraty z = d[v + 1] + MUL15(xu, y[v + 1])
143*0957b409SSimon J. Gerraty + MUL15(f, m[v + 1]) + r;
144*0957b409SSimon J. Gerraty r = z >> 15;
145*0957b409SSimon J. Gerraty d[v + 0] = z & 0x7FFF;
146*0957b409SSimon J. Gerraty z = d[v + 2] + MUL15(xu, y[v + 2])
147*0957b409SSimon J. Gerraty + MUL15(f, m[v + 2]) + r;
148*0957b409SSimon J. Gerraty r = z >> 15;
149*0957b409SSimon J. Gerraty d[v + 1] = z & 0x7FFF;
150*0957b409SSimon J. Gerraty z = d[v + 3] + MUL15(xu, y[v + 3])
151*0957b409SSimon J. Gerraty + MUL15(f, m[v + 3]) + r;
152*0957b409SSimon J. Gerraty r = z >> 15;
153*0957b409SSimon J. Gerraty d[v + 2] = z & 0x7FFF;
154*0957b409SSimon J. Gerraty z = d[v + 4] + MUL15(xu, y[v + 4])
155*0957b409SSimon J. Gerraty + MUL15(f, m[v + 4]) + r;
156*0957b409SSimon J. Gerraty r = z >> 15;
157*0957b409SSimon J. Gerraty d[v + 3] = z & 0x7FFF;
158*0957b409SSimon J. Gerraty }
159*0957b409SSimon J. Gerraty #endif
160*0957b409SSimon J. Gerraty for (; v < len; v ++) {
161*0957b409SSimon J. Gerraty uint32_t z;
162*0957b409SSimon J. Gerraty
163*0957b409SSimon J. Gerraty z = d[v + 1] + MUL15(xu, y[v + 1])
164*0957b409SSimon J. Gerraty + MUL15(f, m[v + 1]) + r;
165*0957b409SSimon J. Gerraty r = z >> 15;
166*0957b409SSimon J. Gerraty d[v + 0] = z & 0x7FFF;
167*0957b409SSimon J. Gerraty }
168*0957b409SSimon J. Gerraty
169*0957b409SSimon J. Gerraty zh = dh + r;
170*0957b409SSimon J. Gerraty d[len] = zh & 0x7FFF;
171*0957b409SSimon J. Gerraty dh = zh >> 15;
172*0957b409SSimon J. Gerraty }
173*0957b409SSimon J. Gerraty
174*0957b409SSimon J. Gerraty /*
175*0957b409SSimon J. Gerraty * Restore the bit length (it was overwritten in the loop above).
176*0957b409SSimon J. Gerraty */
177*0957b409SSimon J. Gerraty d[0] = m[0];
178*0957b409SSimon J. Gerraty
179*0957b409SSimon J. Gerraty /*
180*0957b409SSimon J. Gerraty * d[] may be greater than m[], but it is still lower than twice
181*0957b409SSimon J. Gerraty * the modulus.
182*0957b409SSimon J. Gerraty */
183*0957b409SSimon J. Gerraty br_i15_sub(d, m, NEQ(dh, 0) | NOT(br_i15_sub(d, m, 0)));
184*0957b409SSimon J. Gerraty }
185