1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */
2
3 /*-
4 * Copyright (c) 2001 Theo de Raadt
5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
6 * Copyright (c) 2014-2021 The FreeBSD Foundation
7 * All rights reserved.
8 *
9 * Portions of this software were developed by John-Mark Gurney
10 * under sponsorship of the FreeBSD Foundation and
11 * Rubicon Communications, LLC (Netgate).
12 *
13 * Portions of this software were developed by Ararat River
14 * Consulting, LLC under sponsorship of the FreeBSD Foundation.
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 *
20 * 1. Redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer.
22 * 2. Redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution.
25 * 3. The name of the author may not be used to endorse or promote products
26 * derived from this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
29 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
30 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
31 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
32 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
33 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
37 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 *
39 * Effort sponsored in part by the Defense Advanced Research Projects
40 * Agency (DARPA) and Air Force Research Laboratory, Air Force
41 * Materiel Command, USAF, under agreement number F30602-01-2-0537.
42 */
43
44 #include <sys/param.h>
45 #include <sys/systm.h>
46 #include <sys/malloc.h>
47 #include <sys/mbuf.h>
48 #include <sys/lock.h>
49 #include <sys/mutex.h>
50 #include <sys/proc.h>
51 #include <sys/sysctl.h>
52 #include <sys/errno.h>
53 #include <sys/random.h>
54 #include <sys/conf.h>
55 #include <sys/kernel.h>
56 #include <sys/module.h>
57 #include <sys/fcntl.h>
58 #include <sys/bus.h>
59 #include <sys/sdt.h>
60 #include <sys/syscallsubr.h>
61
62 #include <opencrypto/cryptodev.h>
63 #include <opencrypto/xform.h>
64
65 SDT_PROVIDER_DECLARE(opencrypto);
66
67 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/);
68
69 #ifdef COMPAT_FREEBSD12
70 /*
71 * Previously, most ioctls were performed against a cloned descriptor
72 * of /dev/crypto obtained via CRIOGET. Now all ioctls are performed
73 * against /dev/crypto directly.
74 */
75 #define CRIOGET _IOWR('c', 100, uint32_t)
76 #endif
77
78 /* the following are done against the cloned descriptor */
79
80 #ifdef COMPAT_FREEBSD32
81 #include <sys/mount.h>
82 #include <compat/freebsd32/freebsd32.h>
83
84 struct session_op32 {
85 uint32_t cipher;
86 uint32_t mac;
87 uint32_t keylen;
88 uint32_t key;
89 int mackeylen;
90 uint32_t mackey;
91 uint32_t ses;
92 };
93
94 struct session2_op32 {
95 uint32_t cipher;
96 uint32_t mac;
97 uint32_t keylen;
98 uint32_t key;
99 int mackeylen;
100 uint32_t mackey;
101 uint32_t ses;
102 int crid;
103 int ivlen;
104 int maclen;
105 int pad[2];
106 };
107
108 struct crypt_op32 {
109 uint32_t ses;
110 uint16_t op;
111 uint16_t flags;
112 u_int len;
113 uint32_t src, dst;
114 uint32_t mac;
115 uint32_t iv;
116 };
117
118 struct crypt_aead32 {
119 uint32_t ses;
120 uint16_t op;
121 uint16_t flags;
122 u_int len;
123 u_int aadlen;
124 u_int ivlen;
125 uint32_t src;
126 uint32_t dst;
127 uint32_t aad;
128 uint32_t tag;
129 uint32_t iv;
130 };
131
132 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32)
133 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32)
134 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32)
135 #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32)
136
137 static void
session_op_from_32(const struct session_op32 * from,struct session2_op * to)138 session_op_from_32(const struct session_op32 *from, struct session2_op *to)
139 {
140
141 memset(to, 0, sizeof(*to));
142 CP(*from, *to, cipher);
143 CP(*from, *to, mac);
144 CP(*from, *to, keylen);
145 PTRIN_CP(*from, *to, key);
146 CP(*from, *to, mackeylen);
147 PTRIN_CP(*from, *to, mackey);
148 CP(*from, *to, ses);
149 to->crid = CRYPTOCAP_F_HARDWARE;
150 }
151
152 static void
session2_op_from_32(const struct session2_op32 * from,struct session2_op * to)153 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to)
154 {
155
156 session_op_from_32((const struct session_op32 *)from, to);
157 CP(*from, *to, crid);
158 CP(*from, *to, ivlen);
159 CP(*from, *to, maclen);
160 }
161
162 static void
session_op_to_32(const struct session2_op * from,struct session_op32 * to)163 session_op_to_32(const struct session2_op *from, struct session_op32 *to)
164 {
165
166 CP(*from, *to, cipher);
167 CP(*from, *to, mac);
168 CP(*from, *to, keylen);
169 PTROUT_CP(*from, *to, key);
170 CP(*from, *to, mackeylen);
171 PTROUT_CP(*from, *to, mackey);
172 CP(*from, *to, ses);
173 }
174
175 static void
session2_op_to_32(const struct session2_op * from,struct session2_op32 * to)176 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to)
177 {
178
179 session_op_to_32(from, (struct session_op32 *)to);
180 CP(*from, *to, crid);
181 }
182
183 static void
crypt_op_from_32(const struct crypt_op32 * from,struct crypt_op * to)184 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to)
185 {
186
187 CP(*from, *to, ses);
188 CP(*from, *to, op);
189 CP(*from, *to, flags);
190 CP(*from, *to, len);
191 PTRIN_CP(*from, *to, src);
192 PTRIN_CP(*from, *to, dst);
193 PTRIN_CP(*from, *to, mac);
194 PTRIN_CP(*from, *to, iv);
195 }
196
197 static void
crypt_op_to_32(const struct crypt_op * from,struct crypt_op32 * to)198 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to)
199 {
200
201 CP(*from, *to, ses);
202 CP(*from, *to, op);
203 CP(*from, *to, flags);
204 CP(*from, *to, len);
205 PTROUT_CP(*from, *to, src);
206 PTROUT_CP(*from, *to, dst);
207 PTROUT_CP(*from, *to, mac);
208 PTROUT_CP(*from, *to, iv);
209 }
210
211 static void
crypt_aead_from_32(const struct crypt_aead32 * from,struct crypt_aead * to)212 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to)
213 {
214
215 CP(*from, *to, ses);
216 CP(*from, *to, op);
217 CP(*from, *to, flags);
218 CP(*from, *to, len);
219 CP(*from, *to, aadlen);
220 CP(*from, *to, ivlen);
221 PTRIN_CP(*from, *to, src);
222 PTRIN_CP(*from, *to, dst);
223 PTRIN_CP(*from, *to, aad);
224 PTRIN_CP(*from, *to, tag);
225 PTRIN_CP(*from, *to, iv);
226 }
227
228 static void
crypt_aead_to_32(const struct crypt_aead * from,struct crypt_aead32 * to)229 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to)
230 {
231
232 CP(*from, *to, ses);
233 CP(*from, *to, op);
234 CP(*from, *to, flags);
235 CP(*from, *to, len);
236 CP(*from, *to, aadlen);
237 CP(*from, *to, ivlen);
238 PTROUT_CP(*from, *to, src);
239 PTROUT_CP(*from, *to, dst);
240 PTROUT_CP(*from, *to, aad);
241 PTROUT_CP(*from, *to, tag);
242 PTROUT_CP(*from, *to, iv);
243 }
244 #endif
245
246 static void
session2_op_from_op(const struct session_op * from,struct session2_op * to)247 session2_op_from_op(const struct session_op *from, struct session2_op *to)
248 {
249
250 memset(to, 0, sizeof(*to));
251 memcpy(to, from, sizeof(*from));
252 to->crid = CRYPTOCAP_F_HARDWARE;
253 }
254
255 static void
session2_op_to_op(const struct session2_op * from,struct session_op * to)256 session2_op_to_op(const struct session2_op *from, struct session_op *to)
257 {
258
259 memcpy(to, from, sizeof(*to));
260 }
261
262 struct csession {
263 TAILQ_ENTRY(csession) next;
264 crypto_session_t cses;
265 volatile u_int refs;
266 uint32_t ses;
267 struct mtx lock; /* for op submission */
268
269 u_int blocksize;
270 int hashsize;
271 int ivsize;
272
273 void *key;
274 void *mackey;
275 };
276
277 struct cryptop_data {
278 struct csession *cse;
279
280 char *buf;
281 char *obuf;
282 char *aad;
283 bool done;
284 };
285
286 struct fcrypt {
287 TAILQ_HEAD(csessionlist, csession) csessions;
288 int sesn;
289 struct mtx lock;
290 };
291
292 static bool use_outputbuffers;
293 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW,
294 &use_outputbuffers, 0,
295 "Use separate output buffers for /dev/crypto requests.");
296
297 static bool use_separate_aad;
298 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW,
299 &use_separate_aad, 0,
300 "Use separate AAD buffer for /dev/crypto requests.");
301
302 static MALLOC_DEFINE(M_CRYPTODEV, "cryptodev", "/dev/crypto data buffers");
303
304 /*
305 * Check a crypto identifier to see if it requested
306 * a software device/driver. This can be done either
307 * by device name/class or through search constraints.
308 */
309 static int
checkforsoftware(int * cridp)310 checkforsoftware(int *cridp)
311 {
312 int crid;
313
314 crid = *cridp;
315
316 if (!crypto_devallowsoft) {
317 if (crid & CRYPTOCAP_F_SOFTWARE) {
318 if (crid & CRYPTOCAP_F_HARDWARE) {
319 *cridp = CRYPTOCAP_F_HARDWARE;
320 return 0;
321 }
322 return EINVAL;
323 }
324 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 &&
325 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0)
326 return EINVAL;
327 }
328 return 0;
329 }
330
331 static int
cse_create(struct fcrypt * fcr,struct session2_op * sop)332 cse_create(struct fcrypt *fcr, struct session2_op *sop)
333 {
334 struct crypto_session_params csp;
335 struct csession *cse;
336 const struct enc_xform *txform;
337 const struct auth_hash *thash;
338 void *key = NULL;
339 void *mackey = NULL;
340 crypto_session_t cses;
341 int crid, error, mac;
342
343 mac = sop->mac;
344 #ifdef COMPAT_FREEBSD12
345 switch (sop->mac) {
346 case CRYPTO_AES_128_NIST_GMAC:
347 case CRYPTO_AES_192_NIST_GMAC:
348 case CRYPTO_AES_256_NIST_GMAC:
349 /* Should always be paired with GCM. */
350 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) {
351 CRYPTDEB("GMAC without GCM");
352 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
353 return (EINVAL);
354 }
355 if (sop->keylen != sop->mackeylen) {
356 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
357 return (EINVAL);
358 }
359 mac = 0;
360 break;
361 case CRYPTO_AES_CCM_CBC_MAC:
362 /* Should always be paired with CCM. */
363 if (sop->cipher != CRYPTO_AES_CCM_16) {
364 CRYPTDEB("CBC-MAC without CCM");
365 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
366 return (EINVAL);
367 }
368 if (sop->keylen != sop->mackeylen) {
369 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
370 return (EINVAL);
371 }
372 mac = 0;
373 break;
374 }
375 #endif
376
377 memset(&csp, 0, sizeof(csp));
378 if (use_outputbuffers)
379 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
380 if (mac != 0) {
381 csp.csp_auth_alg = mac;
382 csp.csp_auth_klen = sop->mackeylen;
383 }
384 if (sop->cipher != 0) {
385 csp.csp_cipher_alg = sop->cipher;
386 csp.csp_cipher_klen = sop->keylen;
387 }
388 thash = crypto_auth_hash(&csp);
389 txform = crypto_cipher(&csp);
390
391 if (txform != NULL && txform->macsize != 0) {
392 if (mac != 0) {
393 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
394 return (EINVAL);
395 }
396 csp.csp_mode = CSP_MODE_AEAD;
397 } else if (txform != NULL && thash != NULL) {
398 csp.csp_mode = CSP_MODE_ETA;
399 } else if (txform != NULL) {
400 csp.csp_mode = CSP_MODE_CIPHER;
401 } else if (thash != NULL) {
402 csp.csp_mode = CSP_MODE_DIGEST;
403 } else {
404 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
405 return (EINVAL);
406 }
407
408 switch (csp.csp_mode) {
409 case CSP_MODE_AEAD:
410 case CSP_MODE_ETA:
411 if (use_separate_aad)
412 csp.csp_flags |= CSP_F_SEPARATE_AAD;
413 break;
414 }
415
416 if (txform != NULL) {
417 if (sop->keylen > txform->maxkey ||
418 sop->keylen < txform->minkey) {
419 CRYPTDEB("invalid cipher parameters");
420 error = EINVAL;
421 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
422 goto bail;
423 }
424
425 key = malloc(csp.csp_cipher_klen, M_CRYPTODEV, M_WAITOK);
426 error = copyin(sop->key, key, csp.csp_cipher_klen);
427 if (error) {
428 CRYPTDEB("invalid key");
429 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
430 goto bail;
431 }
432 csp.csp_cipher_key = key;
433 csp.csp_ivlen = txform->ivsize;
434 }
435
436 if (thash != NULL) {
437 if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) {
438 CRYPTDEB("invalid mac key length");
439 error = EINVAL;
440 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
441 goto bail;
442 }
443
444 if (csp.csp_auth_klen != 0) {
445 mackey = malloc(csp.csp_auth_klen, M_CRYPTODEV,
446 M_WAITOK);
447 error = copyin(sop->mackey, mackey, csp.csp_auth_klen);
448 if (error) {
449 CRYPTDEB("invalid mac key");
450 SDT_PROBE1(opencrypto, dev, ioctl, error,
451 __LINE__);
452 goto bail;
453 }
454 csp.csp_auth_key = mackey;
455 }
456
457 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC)
458 csp.csp_ivlen = AES_GCM_IV_LEN;
459 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC)
460 csp.csp_ivlen = AES_CCM_IV_LEN;
461 }
462
463 if (sop->ivlen != 0) {
464 if (csp.csp_ivlen == 0) {
465 CRYPTDEB("does not support an IV");
466 error = EINVAL;
467 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
468 goto bail;
469 }
470 csp.csp_ivlen = sop->ivlen;
471 }
472 if (sop->maclen != 0) {
473 if (!(thash != NULL || csp.csp_mode == CSP_MODE_AEAD)) {
474 CRYPTDEB("does not support a MAC");
475 error = EINVAL;
476 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
477 goto bail;
478 }
479 csp.csp_auth_mlen = sop->maclen;
480 }
481
482 crid = sop->crid;
483 error = checkforsoftware(&crid);
484 if (error) {
485 CRYPTDEB("checkforsoftware");
486 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
487 goto bail;
488 }
489 error = crypto_newsession(&cses, &csp, crid);
490 if (error) {
491 CRYPTDEB("crypto_newsession");
492 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
493 goto bail;
494 }
495
496 cse = malloc(sizeof(struct csession), M_CRYPTODEV, M_WAITOK | M_ZERO);
497 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF);
498 refcount_init(&cse->refs, 1);
499 cse->key = key;
500 cse->mackey = mackey;
501 cse->cses = cses;
502 if (sop->maclen != 0)
503 cse->hashsize = sop->maclen;
504 else if (thash != NULL)
505 cse->hashsize = thash->hashsize;
506 else if (csp.csp_mode == CSP_MODE_AEAD)
507 cse->hashsize = txform->macsize;
508 cse->ivsize = csp.csp_ivlen;
509
510 /*
511 * NB: This isn't necessarily the block size of the underlying
512 * MAC or cipher but is instead a restriction on valid input
513 * sizes.
514 */
515 if (txform != NULL)
516 cse->blocksize = txform->blocksize;
517 else
518 cse->blocksize = 1;
519
520 mtx_lock(&fcr->lock);
521 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next);
522 cse->ses = fcr->sesn++;
523 mtx_unlock(&fcr->lock);
524
525 sop->ses = cse->ses;
526
527 /* return hardware/driver id */
528 sop->crid = crypto_ses2hid(cse->cses);
529 bail:
530 if (error) {
531 free(key, M_CRYPTODEV);
532 free(mackey, M_CRYPTODEV);
533 }
534 return (error);
535 }
536
537 static struct csession *
cse_find(struct fcrypt * fcr,u_int ses)538 cse_find(struct fcrypt *fcr, u_int ses)
539 {
540 struct csession *cse;
541
542 mtx_lock(&fcr->lock);
543 TAILQ_FOREACH(cse, &fcr->csessions, next) {
544 if (cse->ses == ses) {
545 refcount_acquire(&cse->refs);
546 mtx_unlock(&fcr->lock);
547 return (cse);
548 }
549 }
550 mtx_unlock(&fcr->lock);
551 return (NULL);
552 }
553
554 static void
cse_free(struct csession * cse)555 cse_free(struct csession *cse)
556 {
557
558 if (!refcount_release(&cse->refs))
559 return;
560 crypto_freesession(cse->cses);
561 mtx_destroy(&cse->lock);
562 if (cse->key)
563 free(cse->key, M_CRYPTODEV);
564 if (cse->mackey)
565 free(cse->mackey, M_CRYPTODEV);
566 free(cse, M_CRYPTODEV);
567 }
568
569 static bool
cse_delete(struct fcrypt * fcr,u_int ses)570 cse_delete(struct fcrypt *fcr, u_int ses)
571 {
572 struct csession *cse;
573
574 mtx_lock(&fcr->lock);
575 TAILQ_FOREACH(cse, &fcr->csessions, next) {
576 if (cse->ses == ses) {
577 TAILQ_REMOVE(&fcr->csessions, cse, next);
578 mtx_unlock(&fcr->lock);
579 cse_free(cse);
580 return (true);
581 }
582 }
583 mtx_unlock(&fcr->lock);
584 return (false);
585 }
586
587 static struct cryptop_data *
cod_alloc(struct csession * cse,size_t aad_len,size_t len)588 cod_alloc(struct csession *cse, size_t aad_len, size_t len)
589 {
590 struct cryptop_data *cod;
591
592 cod = malloc(sizeof(struct cryptop_data), M_CRYPTODEV, M_WAITOK |
593 M_ZERO);
594
595 cod->cse = cse;
596 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) {
597 if (aad_len != 0)
598 cod->aad = malloc(aad_len, M_CRYPTODEV, M_WAITOK);
599 cod->buf = malloc(len, M_CRYPTODEV, M_WAITOK);
600 } else
601 cod->buf = malloc(aad_len + len, M_CRYPTODEV, M_WAITOK);
602 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT)
603 cod->obuf = malloc(len, M_CRYPTODEV, M_WAITOK);
604 return (cod);
605 }
606
607 static void
cod_free(struct cryptop_data * cod)608 cod_free(struct cryptop_data *cod)
609 {
610
611 free(cod->aad, M_CRYPTODEV);
612 free(cod->obuf, M_CRYPTODEV);
613 free(cod->buf, M_CRYPTODEV);
614 free(cod, M_CRYPTODEV);
615 }
616
617 static int
cryptodev_cb(struct cryptop * crp)618 cryptodev_cb(struct cryptop *crp)
619 {
620 struct cryptop_data *cod = crp->crp_opaque;
621
622 /*
623 * Lock to ensure the wakeup() is not missed by the loops
624 * waiting on cod->done in cryptodev_op() and
625 * cryptodev_aead().
626 */
627 mtx_lock(&cod->cse->lock);
628 cod->done = true;
629 mtx_unlock(&cod->cse->lock);
630 wakeup(cod);
631 return (0);
632 }
633
634 static int
cryptodev_op(struct csession * cse,const struct crypt_op * cop)635 cryptodev_op(struct csession *cse, const struct crypt_op *cop)
636 {
637 const struct crypto_session_params *csp;
638 struct cryptop_data *cod = NULL;
639 struct cryptop *crp = NULL;
640 char *dst;
641 int error;
642
643 if (cop->len > 256*1024-4) {
644 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
645 return (E2BIG);
646 }
647
648 if ((cop->len % cse->blocksize) != 0) {
649 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
650 return (EINVAL);
651 }
652
653 if (cop->mac && cse->hashsize == 0) {
654 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
655 return (EINVAL);
656 }
657
658 /*
659 * The COP_F_CIPHER_FIRST flag predates explicit session
660 * modes, but the only way it was used was for EtA so allow it
661 * as long as it is consistent with EtA.
662 */
663 if (cop->flags & COP_F_CIPHER_FIRST) {
664 if (cop->op != COP_ENCRYPT) {
665 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
666 return (EINVAL);
667 }
668 }
669
670 cod = cod_alloc(cse, 0, cop->len + cse->hashsize);
671 dst = cop->dst;
672
673 crp = crypto_getreq(cse->cses, M_WAITOK);
674
675 error = copyin(cop->src, cod->buf, cop->len);
676 if (error) {
677 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
678 goto bail;
679 }
680 crp->crp_payload_start = 0;
681 crp->crp_payload_length = cop->len;
682 if (cse->hashsize)
683 crp->crp_digest_start = cop->len;
684
685 csp = crypto_get_params(cse->cses);
686 switch (csp->csp_mode) {
687 case CSP_MODE_COMPRESS:
688 switch (cop->op) {
689 case COP_ENCRYPT:
690 crp->crp_op = CRYPTO_OP_COMPRESS;
691 break;
692 case COP_DECRYPT:
693 crp->crp_op = CRYPTO_OP_DECOMPRESS;
694 break;
695 default:
696 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
697 error = EINVAL;
698 goto bail;
699 }
700 break;
701 case CSP_MODE_CIPHER:
702 if (cop->len == 0 ||
703 (cop->iv == NULL && cop->len == cse->ivsize)) {
704 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
705 error = EINVAL;
706 goto bail;
707 }
708 switch (cop->op) {
709 case COP_ENCRYPT:
710 crp->crp_op = CRYPTO_OP_ENCRYPT;
711 break;
712 case COP_DECRYPT:
713 crp->crp_op = CRYPTO_OP_DECRYPT;
714 break;
715 default:
716 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
717 error = EINVAL;
718 goto bail;
719 }
720 break;
721 case CSP_MODE_DIGEST:
722 switch (cop->op) {
723 case 0:
724 case COP_ENCRYPT:
725 case COP_DECRYPT:
726 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
727 if (cod->obuf != NULL)
728 crp->crp_digest_start = 0;
729 break;
730 default:
731 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
732 error = EINVAL;
733 goto bail;
734 }
735 break;
736 case CSP_MODE_AEAD:
737 if (cse->ivsize != 0 && cop->iv == NULL) {
738 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
739 error = EINVAL;
740 goto bail;
741 }
742 /* FALLTHROUGH */
743 case CSP_MODE_ETA:
744 switch (cop->op) {
745 case COP_ENCRYPT:
746 crp->crp_op = CRYPTO_OP_ENCRYPT |
747 CRYPTO_OP_COMPUTE_DIGEST;
748 break;
749 case COP_DECRYPT:
750 crp->crp_op = CRYPTO_OP_DECRYPT |
751 CRYPTO_OP_VERIFY_DIGEST;
752 break;
753 default:
754 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
755 error = EINVAL;
756 goto bail;
757 }
758 break;
759 default:
760 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
761 error = EINVAL;
762 goto bail;
763 }
764
765 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH);
766 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize);
767 if (cod->obuf)
768 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize);
769 crp->crp_callback = cryptodev_cb;
770 crp->crp_opaque = cod;
771
772 if (cop->iv) {
773 if (cse->ivsize == 0) {
774 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
775 error = EINVAL;
776 goto bail;
777 }
778 error = copyin(cop->iv, crp->crp_iv, cse->ivsize);
779 if (error) {
780 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
781 goto bail;
782 }
783 crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
784 } else if (cse->ivsize != 0) {
785 if (crp->crp_payload_length < cse->ivsize) {
786 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
787 error = EINVAL;
788 goto bail;
789 }
790 crp->crp_iv_start = 0;
791 crp->crp_payload_length -= cse->ivsize;
792 if (crp->crp_payload_length != 0)
793 crp->crp_payload_start = cse->ivsize;
794 dst += cse->ivsize;
795 }
796
797 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
798 error = copyin(cop->mac, cod->buf + crp->crp_digest_start,
799 cse->hashsize);
800 if (error) {
801 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
802 goto bail;
803 }
804 }
805 again:
806 /*
807 * Let the dispatch run unlocked, then, interlock against the
808 * callback before checking if the operation completed and going
809 * to sleep. This insures drivers don't inherit our lock which
810 * results in a lock order reversal between crypto_dispatch forced
811 * entry and the crypto_done callback into us.
812 */
813 error = crypto_dispatch(crp);
814 if (error != 0) {
815 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
816 goto bail;
817 }
818
819 mtx_lock(&cse->lock);
820 while (!cod->done)
821 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
822 mtx_unlock(&cse->lock);
823
824 if (crp->crp_etype == EAGAIN) {
825 crp->crp_etype = 0;
826 crp->crp_flags &= ~CRYPTO_F_DONE;
827 cod->done = false;
828 goto again;
829 }
830
831 if (crp->crp_etype != 0) {
832 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
833 error = crp->crp_etype;
834 goto bail;
835 }
836
837 if (cop->dst != NULL) {
838 error = copyout(cod->obuf != NULL ? cod->obuf :
839 cod->buf + crp->crp_payload_start, dst,
840 crp->crp_payload_length);
841 if (error) {
842 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
843 goto bail;
844 }
845 }
846
847 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) {
848 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) +
849 crp->crp_digest_start, cop->mac, cse->hashsize);
850 if (error) {
851 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
852 goto bail;
853 }
854 }
855
856 bail:
857 crypto_freereq(crp);
858 cod_free(cod);
859
860 return (error);
861 }
862
863 static int
cryptodev_aead(struct csession * cse,struct crypt_aead * caead)864 cryptodev_aead(struct csession *cse, struct crypt_aead *caead)
865 {
866 const struct crypto_session_params *csp;
867 struct cryptop_data *cod = NULL;
868 struct cryptop *crp = NULL;
869 char *dst;
870 int error;
871
872 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) {
873 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
874 return (E2BIG);
875 }
876
877 if ((caead->len % cse->blocksize) != 0) {
878 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
879 return (EINVAL);
880 }
881
882 if (cse->hashsize == 0 || caead->tag == NULL) {
883 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
884 return (EINVAL);
885 }
886
887 /*
888 * The COP_F_CIPHER_FIRST flag predates explicit session
889 * modes, but the only way it was used was for EtA so allow it
890 * as long as it is consistent with EtA.
891 */
892 if (caead->flags & COP_F_CIPHER_FIRST) {
893 if (caead->op != COP_ENCRYPT) {
894 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
895 return (EINVAL);
896 }
897 }
898
899 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize);
900 dst = caead->dst;
901
902 crp = crypto_getreq(cse->cses, M_WAITOK);
903
904 if (cod->aad != NULL)
905 error = copyin(caead->aad, cod->aad, caead->aadlen);
906 else
907 error = copyin(caead->aad, cod->buf, caead->aadlen);
908 if (error) {
909 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
910 goto bail;
911 }
912 crp->crp_aad = cod->aad;
913 crp->crp_aad_start = 0;
914 crp->crp_aad_length = caead->aadlen;
915
916 if (cod->aad != NULL)
917 crp->crp_payload_start = 0;
918 else
919 crp->crp_payload_start = caead->aadlen;
920 error = copyin(caead->src, cod->buf + crp->crp_payload_start,
921 caead->len);
922 if (error) {
923 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
924 goto bail;
925 }
926 crp->crp_payload_length = caead->len;
927 if (caead->op == COP_ENCRYPT && cod->obuf != NULL)
928 crp->crp_digest_start = crp->crp_payload_output_start +
929 caead->len;
930 else
931 crp->crp_digest_start = crp->crp_payload_start + caead->len;
932
933 csp = crypto_get_params(cse->cses);
934 switch (csp->csp_mode) {
935 case CSP_MODE_AEAD:
936 case CSP_MODE_ETA:
937 switch (caead->op) {
938 case COP_ENCRYPT:
939 crp->crp_op = CRYPTO_OP_ENCRYPT |
940 CRYPTO_OP_COMPUTE_DIGEST;
941 break;
942 case COP_DECRYPT:
943 crp->crp_op = CRYPTO_OP_DECRYPT |
944 CRYPTO_OP_VERIFY_DIGEST;
945 break;
946 default:
947 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
948 error = EINVAL;
949 goto bail;
950 }
951 break;
952 default:
953 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
954 error = EINVAL;
955 goto bail;
956 }
957
958 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH);
959 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len +
960 cse->hashsize);
961 if (cod->obuf != NULL)
962 crypto_use_output_buf(crp, cod->obuf, caead->len +
963 cse->hashsize);
964 crp->crp_callback = cryptodev_cb;
965 crp->crp_opaque = cod;
966
967 if (caead->iv) {
968 /*
969 * Permit a 16-byte IV for AES-XTS, but only use the
970 * first 8 bytes as a block number.
971 */
972 if (csp->csp_mode == CSP_MODE_ETA &&
973 csp->csp_cipher_alg == CRYPTO_AES_XTS &&
974 caead->ivlen == AES_BLOCK_LEN)
975 caead->ivlen = AES_XTS_IV_LEN;
976
977 if (cse->ivsize == 0) {
978 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
979 error = EINVAL;
980 goto bail;
981 }
982 if (caead->ivlen != cse->ivsize) {
983 error = EINVAL;
984 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
985 goto bail;
986 }
987
988 error = copyin(caead->iv, crp->crp_iv, cse->ivsize);
989 if (error) {
990 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
991 goto bail;
992 }
993 crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
994 } else {
995 error = EINVAL;
996 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
997 goto bail;
998 }
999
1000 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
1001 error = copyin(caead->tag, cod->buf + crp->crp_digest_start,
1002 cse->hashsize);
1003 if (error) {
1004 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1005 goto bail;
1006 }
1007 }
1008 again:
1009 /*
1010 * Let the dispatch run unlocked, then, interlock against the
1011 * callback before checking if the operation completed and going
1012 * to sleep. This insures drivers don't inherit our lock which
1013 * results in a lock order reversal between crypto_dispatch forced
1014 * entry and the crypto_done callback into us.
1015 */
1016 error = crypto_dispatch(crp);
1017 if (error != 0) {
1018 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1019 goto bail;
1020 }
1021
1022 mtx_lock(&cse->lock);
1023 while (!cod->done)
1024 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
1025 mtx_unlock(&cse->lock);
1026
1027 if (crp->crp_etype == EAGAIN) {
1028 crp->crp_etype = 0;
1029 crp->crp_flags &= ~CRYPTO_F_DONE;
1030 cod->done = false;
1031 goto again;
1032 }
1033
1034 if (crp->crp_etype != 0) {
1035 error = crp->crp_etype;
1036 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1037 goto bail;
1038 }
1039
1040 if (caead->dst != NULL) {
1041 error = copyout(cod->obuf != NULL ? cod->obuf :
1042 cod->buf + crp->crp_payload_start, dst,
1043 crp->crp_payload_length);
1044 if (error) {
1045 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1046 goto bail;
1047 }
1048 }
1049
1050 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) {
1051 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) +
1052 crp->crp_digest_start, caead->tag, cse->hashsize);
1053 if (error) {
1054 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1055 goto bail;
1056 }
1057 }
1058
1059 bail:
1060 crypto_freereq(crp);
1061 cod_free(cod);
1062
1063 return (error);
1064 }
1065
1066 static int
cryptodev_find(struct crypt_find_op * find)1067 cryptodev_find(struct crypt_find_op *find)
1068 {
1069 device_t dev;
1070 size_t fnlen = sizeof find->name;
1071
1072 if (find->crid != -1) {
1073 dev = crypto_find_device_byhid(find->crid);
1074 if (dev == NULL)
1075 return (ENOENT);
1076 strncpy(find->name, device_get_nameunit(dev), fnlen);
1077 find->name[fnlen - 1] = '\x0';
1078 } else {
1079 find->name[fnlen - 1] = '\x0';
1080 find->crid = crypto_find_driver(find->name);
1081 if (find->crid == -1)
1082 return (ENOENT);
1083 }
1084 return (0);
1085 }
1086
1087 static void
fcrypt_dtor(void * data)1088 fcrypt_dtor(void *data)
1089 {
1090 struct fcrypt *fcr = data;
1091 struct csession *cse;
1092
1093 while ((cse = TAILQ_FIRST(&fcr->csessions))) {
1094 TAILQ_REMOVE(&fcr->csessions, cse, next);
1095 KASSERT(refcount_load(&cse->refs) == 1,
1096 ("%s: crypto session %p with %d refs", __func__, cse,
1097 refcount_load(&cse->refs)));
1098 cse_free(cse);
1099 }
1100 mtx_destroy(&fcr->lock);
1101 free(fcr, M_CRYPTODEV);
1102 }
1103
1104 static int
crypto_open(struct cdev * dev,int oflags,int devtype,struct thread * td)1105 crypto_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
1106 {
1107 struct fcrypt *fcr;
1108 int error;
1109
1110 fcr = malloc(sizeof(struct fcrypt), M_CRYPTODEV, M_WAITOK | M_ZERO);
1111 TAILQ_INIT(&fcr->csessions);
1112 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF);
1113 error = devfs_set_cdevpriv(fcr, fcrypt_dtor);
1114 if (error)
1115 fcrypt_dtor(fcr);
1116 return (error);
1117 }
1118
1119 static int
crypto_ioctl(struct cdev * dev,u_long cmd,caddr_t data,int flag,struct thread * td)1120 crypto_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag,
1121 struct thread *td)
1122 {
1123 struct fcrypt *fcr;
1124 struct csession *cse;
1125 struct session2_op *sop;
1126 struct crypt_op *cop;
1127 struct crypt_aead *caead;
1128 uint32_t ses;
1129 int error = 0;
1130 union {
1131 struct session2_op sopc;
1132 #ifdef COMPAT_FREEBSD32
1133 struct crypt_op copc;
1134 struct crypt_aead aeadc;
1135 #endif
1136 } thunk;
1137 #ifdef COMPAT_FREEBSD32
1138 u_long cmd32;
1139 void *data32;
1140
1141 cmd32 = 0;
1142 data32 = NULL;
1143 switch (cmd) {
1144 case CIOCGSESSION32:
1145 cmd32 = cmd;
1146 data32 = data;
1147 cmd = CIOCGSESSION;
1148 data = (void *)&thunk.sopc;
1149 session_op_from_32((struct session_op32 *)data32, &thunk.sopc);
1150 break;
1151 case CIOCGSESSION232:
1152 cmd32 = cmd;
1153 data32 = data;
1154 cmd = CIOCGSESSION2;
1155 data = (void *)&thunk.sopc;
1156 session2_op_from_32((struct session2_op32 *)data32,
1157 &thunk.sopc);
1158 break;
1159 case CIOCCRYPT32:
1160 cmd32 = cmd;
1161 data32 = data;
1162 cmd = CIOCCRYPT;
1163 data = (void *)&thunk.copc;
1164 crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc);
1165 break;
1166 case CIOCCRYPTAEAD32:
1167 cmd32 = cmd;
1168 data32 = data;
1169 cmd = CIOCCRYPTAEAD;
1170 data = (void *)&thunk.aeadc;
1171 crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc);
1172 break;
1173 }
1174 #endif
1175
1176 devfs_get_cdevpriv((void **)&fcr);
1177
1178 switch (cmd) {
1179 #ifdef COMPAT_FREEBSD12
1180 case CRIOGET:
1181 /*
1182 * NB: This may fail in cases that the old
1183 * implementation did not if the current process has
1184 * restricted filesystem access (e.g. running in a
1185 * jail that does not expose /dev/crypto or in
1186 * capability mode).
1187 */
1188 error = kern_openat(td, AT_FDCWD, "/dev/crypto", UIO_SYSSPACE,
1189 O_RDWR, 0);
1190 if (error == 0)
1191 *(uint32_t *)data = td->td_retval[0];
1192 break;
1193 #endif
1194 case CIOCGSESSION:
1195 case CIOCGSESSION2:
1196 if (cmd == CIOCGSESSION) {
1197 session2_op_from_op((void *)data, &thunk.sopc);
1198 sop = &thunk.sopc;
1199 } else
1200 sop = (struct session2_op *)data;
1201
1202 error = cse_create(fcr, sop);
1203 if (cmd == CIOCGSESSION && error == 0)
1204 session2_op_to_op(sop, (void *)data);
1205 break;
1206 case CIOCFSESSION:
1207 ses = *(uint32_t *)data;
1208 if (!cse_delete(fcr, ses)) {
1209 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1210 return (EINVAL);
1211 }
1212 break;
1213 case CIOCCRYPT:
1214 cop = (struct crypt_op *)data;
1215 cse = cse_find(fcr, cop->ses);
1216 if (cse == NULL) {
1217 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1218 return (EINVAL);
1219 }
1220 error = cryptodev_op(cse, cop);
1221 cse_free(cse);
1222 break;
1223 case CIOCFINDDEV:
1224 error = cryptodev_find((struct crypt_find_op *)data);
1225 break;
1226 case CIOCCRYPTAEAD:
1227 caead = (struct crypt_aead *)data;
1228 cse = cse_find(fcr, caead->ses);
1229 if (cse == NULL) {
1230 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1231 return (EINVAL);
1232 }
1233 error = cryptodev_aead(cse, caead);
1234 cse_free(cse);
1235 break;
1236 default:
1237 error = EINVAL;
1238 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1239 break;
1240 }
1241
1242 #ifdef COMPAT_FREEBSD32
1243 switch (cmd32) {
1244 case CIOCGSESSION32:
1245 if (error == 0)
1246 session_op_to_32((void *)data, data32);
1247 break;
1248 case CIOCGSESSION232:
1249 if (error == 0)
1250 session2_op_to_32((void *)data, data32);
1251 break;
1252 case CIOCCRYPT32:
1253 if (error == 0)
1254 crypt_op_to_32((void *)data, data32);
1255 break;
1256 case CIOCCRYPTAEAD32:
1257 if (error == 0)
1258 crypt_aead_to_32((void *)data, data32);
1259 break;
1260 }
1261 #endif
1262 return (error);
1263 }
1264
1265 static struct cdevsw crypto_cdevsw = {
1266 .d_version = D_VERSION,
1267 .d_open = crypto_open,
1268 .d_ioctl = crypto_ioctl,
1269 .d_name = "crypto",
1270 };
1271 static struct cdev *crypto_dev;
1272
1273 /*
1274 * Initialization code, both for static and dynamic loading.
1275 */
1276 static int
cryptodev_modevent(module_t mod,int type,void * unused)1277 cryptodev_modevent(module_t mod, int type, void *unused)
1278 {
1279 switch (type) {
1280 case MOD_LOAD:
1281 if (bootverbose)
1282 printf("crypto: <crypto device>\n");
1283 crypto_dev = make_dev(&crypto_cdevsw, 0,
1284 UID_ROOT, GID_WHEEL, 0666,
1285 "crypto");
1286 return 0;
1287 case MOD_UNLOAD:
1288 /*XXX disallow if active sessions */
1289 destroy_dev(crypto_dev);
1290 return 0;
1291 }
1292 return EINVAL;
1293 }
1294
1295 static moduledata_t cryptodev_mod = {
1296 "cryptodev",
1297 cryptodev_modevent,
1298 0
1299 };
1300 MODULE_VERSION(cryptodev, 1);
1301 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
1302 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1);
1303 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1);
1304