1 /*-
2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
4 * Copyright (c) 2014,2016 The FreeBSD Foundation
5 * Copyright (c) 2020 Ampere Computing
6 * All rights reserved.
7 *
8 * Portions of this software were developed by John-Mark Gurney
9 * under sponsorship of the FreeBSD Foundation and
10 * Rubicon Communications, LLC (Netgate).
11 *
12 * This software was developed by Andrew Turner under
13 * sponsorship from the FreeBSD Foundation.
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 */
36
37 /*
38 * This is based on the aesni code.
39 */
40
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/kernel.h>
44 #include <sys/bus.h>
45 #include <sys/endian.h>
46 #include <sys/malloc.h>
47 #include <sys/mbuf.h>
48 #include <sys/module.h>
49 #include <sys/queue.h>
50 #include <sys/smp.h>
51 #include <sys/uio.h>
52
53 #include <machine/vfp.h>
54
55 #include <opencrypto/cryptodev.h>
56 #include <opencrypto/gmac.h>
57 #include <cryptodev_if.h>
58 #include <crypto/armv8/armv8_crypto.h>
59 #include <crypto/rijndael/rijndael.h>
60
61 struct armv8_crypto_softc {
62 int32_t cid;
63 bool has_pmul;
64 };
65
66 static int armv8_crypto_cipher_process(struct armv8_crypto_session *,
67 struct cryptop *);
68
69 MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data");
70
71 static void
armv8_crypto_identify(driver_t * drv,device_t parent)72 armv8_crypto_identify(driver_t *drv, device_t parent)
73 {
74
75 /* NB: order 10 is so we get attached after h/w devices */
76 if (device_find_child(parent, "armv8crypto", -1) == NULL &&
77 BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0)
78 panic("ARMv8 crypto: could not attach");
79 }
80
81 static int
armv8_crypto_probe(device_t dev)82 armv8_crypto_probe(device_t dev)
83 {
84 uint64_t reg;
85 int ret = ENXIO;
86
87 reg = READ_SPECIALREG(id_aa64isar0_el1);
88
89 switch (ID_AA64ISAR0_AES_VAL(reg)) {
90 case ID_AA64ISAR0_AES_BASE:
91 ret = 0;
92 device_set_desc(dev, "AES-CBC,AES-XTS");
93 break;
94 case ID_AA64ISAR0_AES_PMULL:
95 ret = 0;
96 device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM");
97 break;
98 default:
99 break;
100 case ID_AA64ISAR0_AES_NONE:
101 device_printf(dev, "CPU lacks AES instructions\n");
102 break;
103 }
104
105 /* TODO: Check more fields as we support more features */
106
107 return (ret);
108 }
109
110 static int
armv8_crypto_attach(device_t dev)111 armv8_crypto_attach(device_t dev)
112 {
113 struct armv8_crypto_softc *sc;
114 uint64_t reg;
115
116 sc = device_get_softc(dev);
117
118 reg = READ_SPECIALREG(id_aa64isar0_el1);
119
120 if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL)
121 sc->has_pmul = true;
122
123 sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session),
124 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE);
125 if (sc->cid < 0) {
126 device_printf(dev, "Could not get crypto driver id.\n");
127 return (ENOMEM);
128 }
129
130 return (0);
131 }
132
133 static int
armv8_crypto_detach(device_t dev)134 armv8_crypto_detach(device_t dev)
135 {
136 struct armv8_crypto_softc *sc;
137
138 sc = device_get_softc(dev);
139
140 crypto_unregister_all(sc->cid);
141
142 return (0);
143 }
144
145 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)
146
147 static int
armv8_crypto_probesession(device_t dev,const struct crypto_session_params * csp)148 armv8_crypto_probesession(device_t dev,
149 const struct crypto_session_params *csp)
150 {
151 struct armv8_crypto_softc *sc;
152
153 sc = device_get_softc(dev);
154
155 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
156 return (EINVAL);
157
158 switch (csp->csp_mode) {
159 case CSP_MODE_AEAD:
160 switch (csp->csp_cipher_alg) {
161 case CRYPTO_AES_NIST_GCM_16:
162 if (!sc->has_pmul)
163 return (EINVAL);
164 if (csp->csp_auth_mlen != 0 &&
165 csp->csp_auth_mlen != GMAC_DIGEST_LEN)
166 return (EINVAL);
167 switch (csp->csp_cipher_klen * 8) {
168 case 128:
169 case 192:
170 case 256:
171 break;
172 default:
173 return (EINVAL);
174 }
175 break;
176 default:
177 return (EINVAL);
178 }
179 break;
180 case CSP_MODE_CIPHER:
181 switch (csp->csp_cipher_alg) {
182 case CRYPTO_AES_CBC:
183 if (csp->csp_ivlen != AES_BLOCK_LEN)
184 return (EINVAL);
185 switch (csp->csp_cipher_klen * 8) {
186 case 128:
187 case 192:
188 case 256:
189 break;
190 default:
191 return (EINVAL);
192 }
193 break;
194 case CRYPTO_AES_XTS:
195 if (csp->csp_ivlen != AES_XTS_IV_LEN)
196 return (EINVAL);
197 switch (csp->csp_cipher_klen * 8) {
198 case 256:
199 case 512:
200 break;
201 default:
202 return (EINVAL);
203 }
204 break;
205 default:
206 return (EINVAL);
207 }
208 break;
209 default:
210 return (EINVAL);
211 }
212 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
213 }
214
215 static int
armv8_crypto_cipher_setup(struct armv8_crypto_session * ses,const struct crypto_session_params * csp,const uint8_t * key,int keylen)216 armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
217 const struct crypto_session_params *csp, const uint8_t *key, int keylen)
218 {
219 __uint128_val_t H;
220
221 if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
222 keylen /= 2;
223
224 switch (keylen * 8) {
225 case 128:
226 case 192:
227 case 256:
228 break;
229 default:
230 return (EINVAL);
231 }
232
233 fpu_kern_enter(curthread, NULL, FPU_KERN_NORMAL | FPU_KERN_NOCTX);
234
235 aes_v8_set_encrypt_key(key,
236 keylen * 8, &ses->enc_schedule);
237
238 if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) ||
239 (csp->csp_cipher_alg == CRYPTO_AES_CBC))
240 aes_v8_set_decrypt_key(key,
241 keylen * 8, &ses->dec_schedule);
242
243 if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
244 aes_v8_set_encrypt_key(key + keylen, keylen * 8, &ses->xts_schedule);
245
246 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
247 memset(H.c, 0, sizeof(H.c));
248 aes_v8_encrypt(H.c, H.c, &ses->enc_schedule);
249 H.u[0] = bswap64(H.u[0]);
250 H.u[1] = bswap64(H.u[1]);
251 gcm_init_v8(ses->Htable, H.u);
252 }
253
254 fpu_kern_leave(curthread, NULL);
255
256 return (0);
257 }
258
259 static int
armv8_crypto_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)260 armv8_crypto_newsession(device_t dev, crypto_session_t cses,
261 const struct crypto_session_params *csp)
262 {
263 struct armv8_crypto_session *ses;
264 int error;
265
266 ses = crypto_get_driver_session(cses);
267 error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key,
268 csp->csp_cipher_klen);
269 return (error);
270 }
271
272 static int
armv8_crypto_process(device_t dev,struct cryptop * crp,int hint __unused)273 armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused)
274 {
275 struct armv8_crypto_session *ses;
276
277 ses = crypto_get_driver_session(crp->crp_session);
278 crp->crp_etype = armv8_crypto_cipher_process(ses, crp);
279 crypto_done(crp);
280 return (0);
281 }
282
283 static uint8_t *
armv8_crypto_cipher_alloc(struct cryptop * crp,int start,int length,int * allocated)284 armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated)
285 {
286 uint8_t *addr;
287
288 addr = crypto_contiguous_subsegment(crp, start, length);
289 if (addr != NULL) {
290 *allocated = 0;
291 return (addr);
292 }
293 addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT);
294 if (addr != NULL) {
295 *allocated = 1;
296 crypto_copydata(crp, start, length, addr);
297 } else
298 *allocated = 0;
299 return (addr);
300 }
301
302 static int
armv8_crypto_cipher_process(struct armv8_crypto_session * ses,struct cryptop * crp)303 armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
304 struct cryptop *crp)
305 {
306 struct crypto_buffer_cursor fromc, toc;
307 const struct crypto_session_params *csp;
308 uint8_t *authbuf;
309 uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN];
310 int authallocated;
311 int encflag;
312 int error;
313
314 csp = crypto_get_params(crp->crp_session);
315 encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
316
317 authallocated = 0;
318 authbuf = NULL;
319
320 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
321 if (crp->crp_aad != NULL)
322 authbuf = crp->crp_aad;
323 else
324 authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start,
325 crp->crp_aad_length, &authallocated);
326 if (authbuf == NULL)
327 return (ENOMEM);
328 }
329 crypto_cursor_init(&fromc, &crp->crp_buf);
330 crypto_cursor_advance(&fromc, crp->crp_payload_start);
331 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
332 crypto_cursor_init(&toc, &crp->crp_obuf);
333 crypto_cursor_advance(&toc, crp->crp_payload_output_start);
334 } else {
335 crypto_cursor_copy(&fromc, &toc);
336 }
337
338 if (crp->crp_cipher_key != NULL) {
339 armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key,
340 csp->csp_cipher_klen);
341 }
342
343 crypto_read_iv(crp, iv);
344
345 fpu_kern_enter(curthread, NULL, FPU_KERN_NORMAL | FPU_KERN_NOCTX);
346
347 error = 0;
348 switch (csp->csp_cipher_alg) {
349 case CRYPTO_AES_CBC:
350 if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
351 error = EINVAL;
352 break;
353 }
354 if (encflag)
355 armv8_aes_encrypt_cbc(&ses->enc_schedule,
356 crp->crp_payload_length, &fromc, &toc, iv);
357 else
358 armv8_aes_decrypt_cbc(&ses->dec_schedule,
359 crp->crp_payload_length, &fromc, &toc, iv);
360 break;
361 case CRYPTO_AES_XTS:
362 if (encflag)
363 armv8_aes_encrypt_xts(&ses->enc_schedule,
364 &ses->xts_schedule.aes_key, crp->crp_payload_length,
365 &fromc, &toc, iv);
366 else
367 armv8_aes_decrypt_xts(&ses->dec_schedule,
368 &ses->xts_schedule.aes_key, crp->crp_payload_length,
369 &fromc, &toc, iv);
370 break;
371 case CRYPTO_AES_NIST_GCM_16:
372 if (encflag) {
373 memset(tag, 0, sizeof(tag));
374 armv8_aes_encrypt_gcm(&ses->enc_schedule,
375 crp->crp_payload_length, &fromc, &toc,
376 crp->crp_aad_length, authbuf, tag, iv, ses->Htable);
377 crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
378 tag);
379 } else {
380 crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
381 tag);
382 error = armv8_aes_decrypt_gcm(&ses->enc_schedule,
383 crp->crp_payload_length, &fromc, &toc,
384 crp->crp_aad_length, authbuf, tag, iv, ses->Htable);
385 }
386 break;
387 }
388
389 fpu_kern_leave(curthread, NULL);
390
391 if (authallocated)
392 zfree(authbuf, M_ARMV8_CRYPTO);
393 explicit_bzero(iv, sizeof(iv));
394 explicit_bzero(tag, sizeof(tag));
395
396 return (error);
397 }
398
399 static device_method_t armv8_crypto_methods[] = {
400 DEVMETHOD(device_identify, armv8_crypto_identify),
401 DEVMETHOD(device_probe, armv8_crypto_probe),
402 DEVMETHOD(device_attach, armv8_crypto_attach),
403 DEVMETHOD(device_detach, armv8_crypto_detach),
404
405 DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession),
406 DEVMETHOD(cryptodev_newsession, armv8_crypto_newsession),
407 DEVMETHOD(cryptodev_process, armv8_crypto_process),
408
409 DEVMETHOD_END,
410 };
411
412 static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods,
413 sizeof(struct armv8_crypto_softc));
414
415 DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, 0, 0);
416