1 /*
2 * Copyright 2010-2022 The OpenSSL Project Authors. All Rights Reserved.
3 *
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
8 */
9
10 #include <sys/types.h>
11 #include <sys/endian.h>
12 #include <sys/systm.h>
13
14 #include <crypto/openssl/ossl.h>
15 #include <crypto/openssl/ossl_arm.h>
16 #include <crypto/openssl/ossl_aes_gcm.h>
17 #include <crypto/openssl/ossl_cipher.h>
18 #include <crypto/openssl/arm_arch.h>
19
20 #include <opencrypto/cryptodev.h>
21
22 _Static_assert(
23 sizeof(struct ossl_gcm_context) <= sizeof(struct ossl_cipher_context),
24 "ossl_gcm_context too large");
25
26 void AES_encrypt(const void *in, void *out, const void *ks);
27 void AES_set_encrypt_key(const void *key, int keylen, void *ks);
28
29 void gcm_init_neon(__uint128_t Htable[16], const uint64_t Xi[2]);
30 void gcm_gmult_neon(uint64_t Xi[2], const __uint128_t Htable[16]);
31 void gcm_ghash_neon(uint64_t Xi[2], const __uint128_t Htable[16],
32 const void *in, size_t len);
33
34 void ossl_bsaes_ctr32_encrypt_blocks(const unsigned char *in,
35 unsigned char *out, size_t blocks, void *ks, const unsigned char *iv);
36
37 static void
gcm_init(struct ossl_gcm_context * ctx,const void * key,size_t keylen)38 gcm_init(struct ossl_gcm_context *ctx, const void *key, size_t keylen)
39 {
40 memset(&ctx->gcm, 0, sizeof(ctx->gcm));
41 memset(&ctx->aes_ks, 0, sizeof(ctx->aes_ks));
42
43 AES_set_encrypt_key(key, keylen, &ctx->aes_ks);
44 AES_encrypt(ctx->gcm.H.c, ctx->gcm.H.c, &ctx->aes_ks);
45
46 #if BYTE_ORDER == LITTLE_ENDIAN
47 ctx->gcm.H.u[0] = bswap64(ctx->gcm.H.u[0]);
48 ctx->gcm.H.u[1] = bswap64(ctx->gcm.H.u[1]);
49 #endif
50
51 gcm_init_neon(ctx->gcm.Htable, ctx->gcm.H.u);
52 }
53
54 static void
gcm_setiv(struct ossl_gcm_context * ctx,const unsigned char * iv,size_t len)55 gcm_setiv(struct ossl_gcm_context *ctx, const unsigned char *iv, size_t len)
56 {
57 uint32_t ctr;
58
59 KASSERT(len == AES_GCM_IV_LEN,
60 ("%s: invalid IV length %zu", __func__, len));
61
62 ctx->gcm.len.u[0] = 0;
63 ctx->gcm.len.u[1] = 0;
64 ctx->gcm.ares = ctx->gcm.mres = 0;
65
66 memcpy(ctx->gcm.Yi.c, iv, len);
67 ctx->gcm.Yi.c[12] = 0;
68 ctx->gcm.Yi.c[13] = 0;
69 ctx->gcm.Yi.c[14] = 0;
70 ctx->gcm.Yi.c[15] = 1;
71 ctr = 1;
72
73 ctx->gcm.Xi.u[0] = 0;
74 ctx->gcm.Xi.u[1] = 0;
75
76 AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EK0.c, &ctx->aes_ks);
77 ctr++;
78
79 #if BYTE_ORDER == LITTLE_ENDIAN
80 ctx->gcm.Yi.d[3] = bswap32(ctr);
81 #else
82 ctx->gcm.Yi.d[3] = ctr;
83 #endif
84 }
85
86 static int
gcm_finish(struct ossl_gcm_context * ctx,const unsigned char * tag,size_t len)87 gcm_finish(struct ossl_gcm_context *ctx, const unsigned char *tag, size_t len)
88 {
89 uint64_t alen = ctx->gcm.len.u[0] << 3;
90 uint64_t clen = ctx->gcm.len.u[1] << 3;
91
92 if (ctx->gcm.mres || ctx->gcm.ares)
93 gcm_gmult_neon(ctx->gcm.Xi.u, ctx->gcm.Htable);
94
95 #if BYTE_ORDER == LITTLE_ENDIAN
96 alen = bswap64(alen);
97 clen = bswap64(clen);
98 #endif
99
100 ctx->gcm.Xi.u[0] ^= alen;
101 ctx->gcm.Xi.u[1] ^= clen;
102 gcm_gmult_neon(ctx->gcm.Xi.u, ctx->gcm.Htable);
103
104 ctx->gcm.Xi.u[0] ^= ctx->gcm.EK0.u[0];
105 ctx->gcm.Xi.u[1] ^= ctx->gcm.EK0.u[1];
106
107 if (tag != NULL)
108 return timingsafe_bcmp(ctx->gcm.Xi.c, tag, len);
109 return 0;
110 }
111
112 static int
gcm_aad(struct ossl_gcm_context * ctx,const unsigned char * aad,size_t len)113 gcm_aad(struct ossl_gcm_context *ctx, const unsigned char *aad, size_t len)
114 {
115 size_t i;
116 unsigned int n;
117 uint64_t alen = ctx->gcm.len.u[0];
118
119 if (ctx->gcm.len.u[1])
120 return -2;
121
122 alen += len;
123 if (alen > ((uint64_t)1 << 61) || (sizeof(len) == 8 && alen < len))
124 return -1;
125 ctx->gcm.len.u[0] = alen;
126
127 n = ctx->gcm.ares;
128 if (n) {
129 while (n && len) {
130 ctx->gcm.Xi.c[n] ^= *(aad++);
131 --len;
132 n = (n + 1) % 16;
133 }
134 if (n == 0)
135 gcm_gmult_neon(ctx->gcm.Xi.u, ctx->gcm.Htable);
136 else {
137 ctx->gcm.ares = n;
138 return 0;
139 }
140 }
141 if ((i = (len & (size_t)-AES_BLOCK_LEN))) {
142 gcm_ghash_neon(ctx->gcm.Xi.u, ctx->gcm.Htable, aad, i);
143 aad += i;
144 len -= i;
145 }
146 if (len) {
147 n = (unsigned int)len;
148 for (i = 0; i < len; ++i)
149 ctx->gcm.Xi.c[i] ^= aad[i];
150 }
151
152 ctx->gcm.ares = n;
153 return 0;
154 }
155
156 static int
gcm_encrypt(struct ossl_gcm_context * ctx,const unsigned char * in,unsigned char * out,size_t len)157 gcm_encrypt(struct ossl_gcm_context *ctx, const unsigned char *in,
158 unsigned char *out, size_t len)
159 {
160 struct bsaes_key bsks;
161 unsigned int n, ctr, mres;
162 size_t i;
163 uint64_t mlen = ctx->gcm.len.u[1];
164
165 mlen += len;
166 if (mlen > (((uint64_t)1 << 36) - 32) ||
167 (sizeof(len) == 8 && mlen < len))
168 return -1;
169 ctx->gcm.len.u[1] = mlen;
170
171 mres = ctx->gcm.mres;
172
173 if (ctx->gcm.ares) {
174 /* First call to encrypt finalizes GHASH(AAD) */
175 gcm_gmult_neon(ctx->gcm.Xi.u, ctx->gcm.Htable);
176 ctx->gcm.ares = 0;
177 }
178
179 #if BYTE_ORDER == LITTLE_ENDIAN
180 ctr = bswap32(ctx->gcm.Yi.d[3]);
181 #else
182 ctr = ctx->gcm.Yi.d[3];
183 #endif
184
185 n = mres % 16;
186 if (n) {
187 while (n && len) {
188 ctx->gcm.Xi.c[n] ^= *(out++) = *(in++) ^ ctx->gcm.EKi.c[n];
189 --len;
190 n = (n + 1) % 16;
191 }
192 if (n == 0) {
193 gcm_gmult_neon(ctx->gcm.Xi.u, ctx->gcm.Htable);
194 mres = 0;
195 } else {
196 ctx->gcm.mres = n;
197 return 0;
198 }
199 }
200 if ((i = (len & (size_t)-16))) {
201 size_t j = i / 16;
202
203 memcpy(&bsks.ks, &ctx->aes_ks, sizeof(bsks.ks));
204 bsks.converted = 0;
205 ossl_bsaes_ctr32_encrypt_blocks(in, out, j, &bsks,
206 ctx->gcm.Yi.c);
207 ctr += (unsigned int)j;
208 #if BYTE_ORDER == LITTLE_ENDIAN
209 ctx->gcm.Yi.d[3] = bswap32(ctr);
210 #else
211 ctx->gcm.Yi.d[3] = ctr;
212 #endif
213 in += i;
214 len -= i;
215 while (j--) {
216 for (i = 0; i < 16; ++i)
217 ctx->gcm.Xi.c[i] ^= out[i];
218 gcm_gmult_neon(ctx->gcm.Xi.u, ctx->gcm.Htable);
219 out += 16;
220 }
221 }
222 if (len) {
223 AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks);
224 ++ctr;
225 #if BYTE_ORDER == LITTLE_ENDIAN
226 ctx->gcm.Yi.d[3] = bswap32(ctr);
227 #else
228 ctx->gcm.Yi.d[3] = ctr;
229 #endif
230 while (len--) {
231 ctx->gcm.Xi.c[mres++] ^= out[n] = in[n] ^ ctx->gcm.EKi.c[n];
232 ++n;
233 }
234 }
235
236 ctx->gcm.mres = mres;
237 return 0;
238 }
239
240 static int
gcm_decrypt(struct ossl_gcm_context * ctx,const unsigned char * in,unsigned char * out,size_t len)241 gcm_decrypt(struct ossl_gcm_context *ctx, const unsigned char *in,
242 unsigned char *out, size_t len)
243 {
244 struct bsaes_key bsks;
245 unsigned int n, ctr, mres;
246 size_t i;
247 uint64_t mlen = ctx->gcm.len.u[1];
248
249 mlen += len;
250 if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len))
251 return -1;
252 ctx->gcm.len.u[1] = mlen;
253
254 mres = ctx->gcm.mres;
255
256 if (ctx->gcm.ares) {
257 /* First call to decrypt finalizes GHASH(AAD) */
258 gcm_gmult_neon(ctx->gcm.Xi.u, ctx->gcm.Htable);
259 ctx->gcm.ares = 0;
260 }
261
262 #if BYTE_ORDER == LITTLE_ENDIAN
263 ctr = bswap32(ctx->gcm.Yi.d[3]);
264 #else
265 ctr = ctx->gcm.Yi.d[3];
266 #endif
267
268 n = mres % 16;
269 if (n) {
270 while (n && len) {
271 uint8_t c = *(in++);
272 *(out++) = c ^ ctx->gcm.EKi.c[n];
273 ctx->gcm.Xi.c[n] ^= c;
274 --len;
275 n = (n + 1) % 16;
276 }
277 if (n == 0) {
278 gcm_gmult_neon(ctx->gcm.Xi.u, ctx->gcm.Htable);
279 mres = 0;
280 } else {
281 ctx->gcm.mres = n;
282 return 0;
283 }
284 }
285 if ((i = (len & (size_t)-16))) {
286 size_t j = i / 16;
287
288 while (j--) {
289 size_t k;
290 for (k = 0; k < 16; ++k)
291 ctx->gcm.Xi.c[k] ^= in[k];
292 gcm_gmult_neon(ctx->gcm.Xi.u, ctx->gcm.Htable);
293 in += 16;
294 }
295 j = i / 16;
296 in -= i;
297 memcpy(&bsks.ks, &ctx->aes_ks, sizeof(bsks.ks));
298 bsks.converted = 0;
299 ossl_bsaes_ctr32_encrypt_blocks(in, out, j, &bsks,
300 ctx->gcm.Yi.c);
301 ctr += (unsigned int)j;
302 #if BYTE_ORDER == LITTLE_ENDIAN
303 ctx->gcm.Yi.d[3] = bswap32(ctr);
304 #else
305 ctx->gcm.Yi.d[3] = ctr;
306 #endif
307 out += i;
308 in += i;
309 len -= i;
310 }
311 if (len) {
312 AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks);
313 ++ctr;
314 #if BYTE_ORDER == LITTLE_ENDIAN
315 ctx->gcm.Yi.d[3] = bswap32(ctr);
316 #else
317 ctx->gcm.Yi.d[3] = ctr;
318 #endif
319 while (len--) {
320 uint8_t c = in[n];
321 ctx->gcm.Xi.c[mres++] ^= c;
322 out[n] = c ^ ctx->gcm.EKi.c[n];
323 ++n;
324 }
325 }
326
327 ctx->gcm.mres = mres;
328 return 0;
329 }
330
331 static void
gcm_tag(struct ossl_gcm_context * ctx,unsigned char * tag,size_t len)332 gcm_tag(struct ossl_gcm_context *ctx, unsigned char *tag, size_t len)
333 {
334 gcm_finish(ctx, NULL, 0);
335 memcpy(tag, ctx->gcm.Xi.c, len);
336 }
337
338 static const struct ossl_aes_gcm_ops gcm_ops_neon = {
339 .init = gcm_init,
340 .setiv = gcm_setiv,
341 .aad = gcm_aad,
342 .encrypt = gcm_encrypt,
343 .decrypt = gcm_decrypt,
344 .finish = gcm_finish,
345 .tag = gcm_tag,
346 };
347
348 int ossl_aes_gcm_setkey(const unsigned char *key, int klen, void *_ctx);
349
350 int
ossl_aes_gcm_setkey(const unsigned char * key,int klen,void * _ctx)351 ossl_aes_gcm_setkey(const unsigned char *key, int klen, void *_ctx)
352 {
353 struct ossl_gcm_context *ctx;
354
355 ctx = _ctx;
356 ctx->ops = &gcm_ops_neon;
357 gcm_init(ctx, key, klen);
358 return (0);
359 }
360