1*0957b409SSimon J. Gerraty /*
2*0957b409SSimon J. Gerraty * Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
3*0957b409SSimon J. Gerraty *
4*0957b409SSimon J. Gerraty * Permission is hereby granted, free of charge, to any person obtaining
5*0957b409SSimon J. Gerraty * a copy of this software and associated documentation files (the
6*0957b409SSimon J. Gerraty * "Software"), to deal in the Software without restriction, including
7*0957b409SSimon J. Gerraty * without limitation the rights to use, copy, modify, merge, publish,
8*0957b409SSimon J. Gerraty * distribute, sublicense, and/or sell copies of the Software, and to
9*0957b409SSimon J. Gerraty * permit persons to whom the Software is furnished to do so, subject to
10*0957b409SSimon J. Gerraty * the following conditions:
11*0957b409SSimon J. Gerraty *
12*0957b409SSimon J. Gerraty * The above copyright notice and this permission notice shall be
13*0957b409SSimon J. Gerraty * included in all copies or substantial portions of the Software.
14*0957b409SSimon J. Gerraty *
15*0957b409SSimon J. Gerraty * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16*0957b409SSimon J. Gerraty * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17*0957b409SSimon J. Gerraty * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18*0957b409SSimon J. Gerraty * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
19*0957b409SSimon J. Gerraty * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
20*0957b409SSimon J. Gerraty * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
21*0957b409SSimon J. Gerraty * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22*0957b409SSimon J. Gerraty * SOFTWARE.
23*0957b409SSimon J. Gerraty */
24*0957b409SSimon J. Gerraty
25*0957b409SSimon J. Gerraty #define BR_ENABLE_INTRINSICS 1
26*0957b409SSimon J. Gerraty #include "inner.h"
27*0957b409SSimon J. Gerraty
28*0957b409SSimon J. Gerraty #if BR_SSE2
29*0957b409SSimon J. Gerraty
30*0957b409SSimon J. Gerraty /*
31*0957b409SSimon J. Gerraty * This file contains a ChaCha20 implementation that leverages SSE2
32*0957b409SSimon J. Gerraty * opcodes for better performance.
33*0957b409SSimon J. Gerraty */
34*0957b409SSimon J. Gerraty
35*0957b409SSimon J. Gerraty /* see bearssl_block.h */
36*0957b409SSimon J. Gerraty br_chacha20_run
br_chacha20_sse2_get(void)37*0957b409SSimon J. Gerraty br_chacha20_sse2_get(void)
38*0957b409SSimon J. Gerraty {
39*0957b409SSimon J. Gerraty /*
40*0957b409SSimon J. Gerraty * If using 64-bit mode, then SSE2 opcodes should be automatically
41*0957b409SSimon J. Gerraty * available, since they are part of the ABI.
42*0957b409SSimon J. Gerraty *
43*0957b409SSimon J. Gerraty * In 32-bit mode, we use CPUID to detect the SSE2 feature.
44*0957b409SSimon J. Gerraty */
45*0957b409SSimon J. Gerraty
46*0957b409SSimon J. Gerraty #if BR_amd64
47*0957b409SSimon J. Gerraty return &br_chacha20_sse2_run;
48*0957b409SSimon J. Gerraty #else
49*0957b409SSimon J. Gerraty
50*0957b409SSimon J. Gerraty /*
51*0957b409SSimon J. Gerraty * SSE2 support is indicated by bit 26 in EDX.
52*0957b409SSimon J. Gerraty */
53*0957b409SSimon J. Gerraty if (br_cpuid(0, 0, 0, 0x04000000)) {
54*0957b409SSimon J. Gerraty return &br_chacha20_sse2_run;
55*0957b409SSimon J. Gerraty } else {
56*0957b409SSimon J. Gerraty return 0;
57*0957b409SSimon J. Gerraty }
58*0957b409SSimon J. Gerraty #endif
59*0957b409SSimon J. Gerraty }
60*0957b409SSimon J. Gerraty
61*0957b409SSimon J. Gerraty BR_TARGETS_X86_UP
62*0957b409SSimon J. Gerraty
63*0957b409SSimon J. Gerraty /* see bearssl_block.h */
64*0957b409SSimon J. Gerraty BR_TARGET("sse2")
65*0957b409SSimon J. Gerraty uint32_t
br_chacha20_sse2_run(const void * key,const void * iv,uint32_t cc,void * data,size_t len)66*0957b409SSimon J. Gerraty br_chacha20_sse2_run(const void *key,
67*0957b409SSimon J. Gerraty const void *iv, uint32_t cc, void *data, size_t len)
68*0957b409SSimon J. Gerraty {
69*0957b409SSimon J. Gerraty unsigned char *buf;
70*0957b409SSimon J. Gerraty uint32_t ivtmp[4];
71*0957b409SSimon J. Gerraty __m128i kw0, kw1;
72*0957b409SSimon J. Gerraty __m128i iw, cw;
73*0957b409SSimon J. Gerraty __m128i one;
74*0957b409SSimon J. Gerraty
75*0957b409SSimon J. Gerraty static const uint32_t CW[] = {
76*0957b409SSimon J. Gerraty 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574
77*0957b409SSimon J. Gerraty };
78*0957b409SSimon J. Gerraty
79*0957b409SSimon J. Gerraty buf = data;
80*0957b409SSimon J. Gerraty kw0 = _mm_loadu_si128(key);
81*0957b409SSimon J. Gerraty kw1 = _mm_loadu_si128((const void *)((const unsigned char *)key + 16));
82*0957b409SSimon J. Gerraty ivtmp[0] = cc;
83*0957b409SSimon J. Gerraty memcpy(ivtmp + 1, iv, 12);
84*0957b409SSimon J. Gerraty iw = _mm_loadu_si128((const void *)ivtmp);
85*0957b409SSimon J. Gerraty cw = _mm_loadu_si128((const void *)CW);
86*0957b409SSimon J. Gerraty one = _mm_set_epi32(0, 0, 0, 1);
87*0957b409SSimon J. Gerraty
88*0957b409SSimon J. Gerraty while (len > 0) {
89*0957b409SSimon J. Gerraty /*
90*0957b409SSimon J. Gerraty * sj contains state words 4*j to 4*j+3.
91*0957b409SSimon J. Gerraty */
92*0957b409SSimon J. Gerraty __m128i s0, s1, s2, s3;
93*0957b409SSimon J. Gerraty int i;
94*0957b409SSimon J. Gerraty
95*0957b409SSimon J. Gerraty s0 = cw;
96*0957b409SSimon J. Gerraty s1 = kw0;
97*0957b409SSimon J. Gerraty s2 = kw1;
98*0957b409SSimon J. Gerraty s3 = iw;
99*0957b409SSimon J. Gerraty for (i = 0; i < 10; i ++) {
100*0957b409SSimon J. Gerraty /*
101*0957b409SSimon J. Gerraty * Even round is straightforward application on
102*0957b409SSimon J. Gerraty * the state words.
103*0957b409SSimon J. Gerraty */
104*0957b409SSimon J. Gerraty s0 = _mm_add_epi32(s0, s1);
105*0957b409SSimon J. Gerraty s3 = _mm_xor_si128(s3, s0);
106*0957b409SSimon J. Gerraty s3 = _mm_or_si128(
107*0957b409SSimon J. Gerraty _mm_slli_epi32(s3, 16),
108*0957b409SSimon J. Gerraty _mm_srli_epi32(s3, 16));
109*0957b409SSimon J. Gerraty
110*0957b409SSimon J. Gerraty s2 = _mm_add_epi32(s2, s3);
111*0957b409SSimon J. Gerraty s1 = _mm_xor_si128(s1, s2);
112*0957b409SSimon J. Gerraty s1 = _mm_or_si128(
113*0957b409SSimon J. Gerraty _mm_slli_epi32(s1, 12),
114*0957b409SSimon J. Gerraty _mm_srli_epi32(s1, 20));
115*0957b409SSimon J. Gerraty
116*0957b409SSimon J. Gerraty s0 = _mm_add_epi32(s0, s1);
117*0957b409SSimon J. Gerraty s3 = _mm_xor_si128(s3, s0);
118*0957b409SSimon J. Gerraty s3 = _mm_or_si128(
119*0957b409SSimon J. Gerraty _mm_slli_epi32(s3, 8),
120*0957b409SSimon J. Gerraty _mm_srli_epi32(s3, 24));
121*0957b409SSimon J. Gerraty
122*0957b409SSimon J. Gerraty s2 = _mm_add_epi32(s2, s3);
123*0957b409SSimon J. Gerraty s1 = _mm_xor_si128(s1, s2);
124*0957b409SSimon J. Gerraty s1 = _mm_or_si128(
125*0957b409SSimon J. Gerraty _mm_slli_epi32(s1, 7),
126*0957b409SSimon J. Gerraty _mm_srli_epi32(s1, 25));
127*0957b409SSimon J. Gerraty
128*0957b409SSimon J. Gerraty /*
129*0957b409SSimon J. Gerraty * For the odd round, we must rotate some state
130*0957b409SSimon J. Gerraty * words so that the computations apply on the
131*0957b409SSimon J. Gerraty * right combinations of words.
132*0957b409SSimon J. Gerraty */
133*0957b409SSimon J. Gerraty s1 = _mm_shuffle_epi32(s1, 0x39);
134*0957b409SSimon J. Gerraty s2 = _mm_shuffle_epi32(s2, 0x4E);
135*0957b409SSimon J. Gerraty s3 = _mm_shuffle_epi32(s3, 0x93);
136*0957b409SSimon J. Gerraty
137*0957b409SSimon J. Gerraty s0 = _mm_add_epi32(s0, s1);
138*0957b409SSimon J. Gerraty s3 = _mm_xor_si128(s3, s0);
139*0957b409SSimon J. Gerraty s3 = _mm_or_si128(
140*0957b409SSimon J. Gerraty _mm_slli_epi32(s3, 16),
141*0957b409SSimon J. Gerraty _mm_srli_epi32(s3, 16));
142*0957b409SSimon J. Gerraty
143*0957b409SSimon J. Gerraty s2 = _mm_add_epi32(s2, s3);
144*0957b409SSimon J. Gerraty s1 = _mm_xor_si128(s1, s2);
145*0957b409SSimon J. Gerraty s1 = _mm_or_si128(
146*0957b409SSimon J. Gerraty _mm_slli_epi32(s1, 12),
147*0957b409SSimon J. Gerraty _mm_srli_epi32(s1, 20));
148*0957b409SSimon J. Gerraty
149*0957b409SSimon J. Gerraty s0 = _mm_add_epi32(s0, s1);
150*0957b409SSimon J. Gerraty s3 = _mm_xor_si128(s3, s0);
151*0957b409SSimon J. Gerraty s3 = _mm_or_si128(
152*0957b409SSimon J. Gerraty _mm_slli_epi32(s3, 8),
153*0957b409SSimon J. Gerraty _mm_srli_epi32(s3, 24));
154*0957b409SSimon J. Gerraty
155*0957b409SSimon J. Gerraty s2 = _mm_add_epi32(s2, s3);
156*0957b409SSimon J. Gerraty s1 = _mm_xor_si128(s1, s2);
157*0957b409SSimon J. Gerraty s1 = _mm_or_si128(
158*0957b409SSimon J. Gerraty _mm_slli_epi32(s1, 7),
159*0957b409SSimon J. Gerraty _mm_srli_epi32(s1, 25));
160*0957b409SSimon J. Gerraty
161*0957b409SSimon J. Gerraty /*
162*0957b409SSimon J. Gerraty * After the odd round, we rotate back the values
163*0957b409SSimon J. Gerraty * to undo the rotate at the start of the odd round.
164*0957b409SSimon J. Gerraty */
165*0957b409SSimon J. Gerraty s1 = _mm_shuffle_epi32(s1, 0x93);
166*0957b409SSimon J. Gerraty s2 = _mm_shuffle_epi32(s2, 0x4E);
167*0957b409SSimon J. Gerraty s3 = _mm_shuffle_epi32(s3, 0x39);
168*0957b409SSimon J. Gerraty }
169*0957b409SSimon J. Gerraty
170*0957b409SSimon J. Gerraty /*
171*0957b409SSimon J. Gerraty * Addition with the initial state.
172*0957b409SSimon J. Gerraty */
173*0957b409SSimon J. Gerraty s0 = _mm_add_epi32(s0, cw);
174*0957b409SSimon J. Gerraty s1 = _mm_add_epi32(s1, kw0);
175*0957b409SSimon J. Gerraty s2 = _mm_add_epi32(s2, kw1);
176*0957b409SSimon J. Gerraty s3 = _mm_add_epi32(s3, iw);
177*0957b409SSimon J. Gerraty
178*0957b409SSimon J. Gerraty /*
179*0957b409SSimon J. Gerraty * Increment block counter.
180*0957b409SSimon J. Gerraty */
181*0957b409SSimon J. Gerraty iw = _mm_add_epi32(iw, one);
182*0957b409SSimon J. Gerraty
183*0957b409SSimon J. Gerraty /*
184*0957b409SSimon J. Gerraty * XOR final state with the data.
185*0957b409SSimon J. Gerraty */
186*0957b409SSimon J. Gerraty if (len < 64) {
187*0957b409SSimon J. Gerraty unsigned char tmp[64];
188*0957b409SSimon J. Gerraty size_t u;
189*0957b409SSimon J. Gerraty
190*0957b409SSimon J. Gerraty _mm_storeu_si128((void *)(tmp + 0), s0);
191*0957b409SSimon J. Gerraty _mm_storeu_si128((void *)(tmp + 16), s1);
192*0957b409SSimon J. Gerraty _mm_storeu_si128((void *)(tmp + 32), s2);
193*0957b409SSimon J. Gerraty _mm_storeu_si128((void *)(tmp + 48), s3);
194*0957b409SSimon J. Gerraty for (u = 0; u < len; u ++) {
195*0957b409SSimon J. Gerraty buf[u] ^= tmp[u];
196*0957b409SSimon J. Gerraty }
197*0957b409SSimon J. Gerraty break;
198*0957b409SSimon J. Gerraty } else {
199*0957b409SSimon J. Gerraty __m128i b0, b1, b2, b3;
200*0957b409SSimon J. Gerraty
201*0957b409SSimon J. Gerraty b0 = _mm_loadu_si128((const void *)(buf + 0));
202*0957b409SSimon J. Gerraty b1 = _mm_loadu_si128((const void *)(buf + 16));
203*0957b409SSimon J. Gerraty b2 = _mm_loadu_si128((const void *)(buf + 32));
204*0957b409SSimon J. Gerraty b3 = _mm_loadu_si128((const void *)(buf + 48));
205*0957b409SSimon J. Gerraty b0 = _mm_xor_si128(b0, s0);
206*0957b409SSimon J. Gerraty b1 = _mm_xor_si128(b1, s1);
207*0957b409SSimon J. Gerraty b2 = _mm_xor_si128(b2, s2);
208*0957b409SSimon J. Gerraty b3 = _mm_xor_si128(b3, s3);
209*0957b409SSimon J. Gerraty _mm_storeu_si128((void *)(buf + 0), b0);
210*0957b409SSimon J. Gerraty _mm_storeu_si128((void *)(buf + 16), b1);
211*0957b409SSimon J. Gerraty _mm_storeu_si128((void *)(buf + 32), b2);
212*0957b409SSimon J. Gerraty _mm_storeu_si128((void *)(buf + 48), b3);
213*0957b409SSimon J. Gerraty buf += 64;
214*0957b409SSimon J. Gerraty len -= 64;
215*0957b409SSimon J. Gerraty }
216*0957b409SSimon J. Gerraty }
217*0957b409SSimon J. Gerraty
218*0957b409SSimon J. Gerraty /*
219*0957b409SSimon J. Gerraty * _mm_extract_epi32() requires SSE4.1. We prefer to stick to
220*0957b409SSimon J. Gerraty * raw SSE2, thus we use _mm_extract_epi16().
221*0957b409SSimon J. Gerraty */
222*0957b409SSimon J. Gerraty return (uint32_t)_mm_extract_epi16(iw, 0)
223*0957b409SSimon J. Gerraty | ((uint32_t)_mm_extract_epi16(iw, 1) << 16);
224*0957b409SSimon J. Gerraty }
225*0957b409SSimon J. Gerraty
226*0957b409SSimon J. Gerraty BR_TARGETS_X86_DOWN
227*0957b409SSimon J. Gerraty
228*0957b409SSimon J. Gerraty #else
229*0957b409SSimon J. Gerraty
230*0957b409SSimon J. Gerraty /* see bearssl_block.h */
231*0957b409SSimon J. Gerraty br_chacha20_run
232*0957b409SSimon J. Gerraty br_chacha20_sse2_get(void)
233*0957b409SSimon J. Gerraty {
234*0957b409SSimon J. Gerraty return 0;
235*0957b409SSimon J. Gerraty }
236*0957b409SSimon J. Gerraty
237*0957b409SSimon J. Gerraty #endif
238