1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */
2
3 /*-
4 * Copyright (c) 2001 Theo de Raadt
5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
6 * Copyright (c) 2014-2021 The FreeBSD Foundation
7 * All rights reserved.
8 *
9 * Portions of this software were developed by John-Mark Gurney
10 * under sponsorship of the FreeBSD Foundation and
11 * Rubicon Communications, LLC (Netgate).
12 *
13 * Portions of this software were developed by Ararat River
14 * Consulting, LLC under sponsorship of the FreeBSD Foundation.
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 *
20 * 1. Redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer.
22 * 2. Redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution.
25 * 3. The name of the author may not be used to endorse or promote products
26 * derived from this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
29 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
30 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
31 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
32 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
33 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
37 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 *
39 * Effort sponsored in part by the Defense Advanced Research Projects
40 * Agency (DARPA) and Air Force Research Laboratory, Air Force
41 * Materiel Command, USAF, under agreement number F30602-01-2-0537.
42 */
43
44 #include <sys/param.h>
45 #include <sys/systm.h>
46 #include <sys/malloc.h>
47 #include <sys/mbuf.h>
48 #include <sys/lock.h>
49 #include <sys/mutex.h>
50 #include <sys/proc.h>
51 #include <sys/sysctl.h>
52 #include <sys/errno.h>
53 #include <sys/random.h>
54 #include <sys/conf.h>
55 #include <sys/kernel.h>
56 #include <sys/module.h>
57 #include <sys/fcntl.h>
58 #include <sys/bus.h>
59 #include <sys/sdt.h>
60 #include <sys/syscallsubr.h>
61
62 #include <opencrypto/cryptodev.h>
63 #include <opencrypto/xform.h>
64
65 SDT_PROVIDER_DECLARE(opencrypto);
66
67 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/);
68
69 #ifdef COMPAT_FREEBSD12
70 /*
71 * Previously, most ioctls were performed against a cloned descriptor
72 * of /dev/crypto obtained via CRIOGET. Now all ioctls are performed
73 * against /dev/crypto directly.
74 */
75 #define CRIOGET _IOWR('c', 100, uint32_t)
76 #endif
77
78 /* the following are done against the cloned descriptor */
79
80 #ifdef COMPAT_FREEBSD32
81 #include <sys/mount.h>
82 #include <compat/freebsd32/freebsd32.h>
83
84 struct session_op32 {
85 uint32_t cipher;
86 uint32_t mac;
87 uint32_t keylen;
88 uint32_t key;
89 int mackeylen;
90 uint32_t mackey;
91 uint32_t ses;
92 };
93
94 struct session2_op32 {
95 uint32_t cipher;
96 uint32_t mac;
97 uint32_t keylen;
98 uint32_t key;
99 int mackeylen;
100 uint32_t mackey;
101 uint32_t ses;
102 int crid;
103 int ivlen;
104 int maclen;
105 int pad[2];
106 };
107
108 struct crypt_op32 {
109 uint32_t ses;
110 uint16_t op;
111 uint16_t flags;
112 u_int len;
113 uint32_t src, dst;
114 uint32_t mac;
115 uint32_t iv;
116 };
117
118 struct crypt_aead32 {
119 uint32_t ses;
120 uint16_t op;
121 uint16_t flags;
122 u_int len;
123 u_int aadlen;
124 u_int ivlen;
125 uint32_t src;
126 uint32_t dst;
127 uint32_t aad;
128 uint32_t tag;
129 uint32_t iv;
130 };
131
132 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32)
133 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32)
134 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32)
135 #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32)
136
137 static void
session_op_from_32(const struct session_op32 * from,struct session2_op * to)138 session_op_from_32(const struct session_op32 *from, struct session2_op *to)
139 {
140
141 memset(to, 0, sizeof(*to));
142 CP(*from, *to, cipher);
143 CP(*from, *to, mac);
144 CP(*from, *to, keylen);
145 PTRIN_CP(*from, *to, key);
146 CP(*from, *to, mackeylen);
147 PTRIN_CP(*from, *to, mackey);
148 CP(*from, *to, ses);
149 to->crid = CRYPTOCAP_F_HARDWARE;
150 }
151
152 static void
session2_op_from_32(const struct session2_op32 * from,struct session2_op * to)153 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to)
154 {
155
156 session_op_from_32((const struct session_op32 *)from, to);
157 CP(*from, *to, crid);
158 CP(*from, *to, ivlen);
159 CP(*from, *to, maclen);
160 }
161
162 static void
session_op_to_32(const struct session2_op * from,struct session_op32 * to)163 session_op_to_32(const struct session2_op *from, struct session_op32 *to)
164 {
165
166 CP(*from, *to, cipher);
167 CP(*from, *to, mac);
168 CP(*from, *to, keylen);
169 PTROUT_CP(*from, *to, key);
170 CP(*from, *to, mackeylen);
171 PTROUT_CP(*from, *to, mackey);
172 CP(*from, *to, ses);
173 }
174
175 static void
session2_op_to_32(const struct session2_op * from,struct session2_op32 * to)176 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to)
177 {
178
179 session_op_to_32(from, (struct session_op32 *)to);
180 CP(*from, *to, crid);
181 }
182
183 static void
crypt_op_from_32(const struct crypt_op32 * from,struct crypt_op * to)184 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to)
185 {
186
187 CP(*from, *to, ses);
188 CP(*from, *to, op);
189 CP(*from, *to, flags);
190 CP(*from, *to, len);
191 PTRIN_CP(*from, *to, src);
192 PTRIN_CP(*from, *to, dst);
193 PTRIN_CP(*from, *to, mac);
194 PTRIN_CP(*from, *to, iv);
195 }
196
197 static void
crypt_op_to_32(const struct crypt_op * from,struct crypt_op32 * to)198 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to)
199 {
200
201 CP(*from, *to, ses);
202 CP(*from, *to, op);
203 CP(*from, *to, flags);
204 CP(*from, *to, len);
205 PTROUT_CP(*from, *to, src);
206 PTROUT_CP(*from, *to, dst);
207 PTROUT_CP(*from, *to, mac);
208 PTROUT_CP(*from, *to, iv);
209 }
210
211 static void
crypt_aead_from_32(const struct crypt_aead32 * from,struct crypt_aead * to)212 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to)
213 {
214
215 CP(*from, *to, ses);
216 CP(*from, *to, op);
217 CP(*from, *to, flags);
218 CP(*from, *to, len);
219 CP(*from, *to, aadlen);
220 CP(*from, *to, ivlen);
221 PTRIN_CP(*from, *to, src);
222 PTRIN_CP(*from, *to, dst);
223 PTRIN_CP(*from, *to, aad);
224 PTRIN_CP(*from, *to, tag);
225 PTRIN_CP(*from, *to, iv);
226 }
227
228 static void
crypt_aead_to_32(const struct crypt_aead * from,struct crypt_aead32 * to)229 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to)
230 {
231
232 CP(*from, *to, ses);
233 CP(*from, *to, op);
234 CP(*from, *to, flags);
235 CP(*from, *to, len);
236 CP(*from, *to, aadlen);
237 CP(*from, *to, ivlen);
238 PTROUT_CP(*from, *to, src);
239 PTROUT_CP(*from, *to, dst);
240 PTROUT_CP(*from, *to, aad);
241 PTROUT_CP(*from, *to, tag);
242 PTROUT_CP(*from, *to, iv);
243 }
244 #endif
245
246 static void
session2_op_from_op(const struct session_op * from,struct session2_op * to)247 session2_op_from_op(const struct session_op *from, struct session2_op *to)
248 {
249
250 memset(to, 0, sizeof(*to));
251 memcpy(to, from, sizeof(*from));
252 to->crid = CRYPTOCAP_F_HARDWARE;
253 }
254
255 static void
session2_op_to_op(const struct session2_op * from,struct session_op * to)256 session2_op_to_op(const struct session2_op *from, struct session_op *to)
257 {
258
259 memcpy(to, from, sizeof(*to));
260 }
261
262 struct csession {
263 TAILQ_ENTRY(csession) next;
264 crypto_session_t cses;
265 volatile u_int refs;
266 uint32_t ses;
267 struct mtx lock; /* for op submission */
268
269 u_int blocksize;
270 int hashsize;
271 int ivsize;
272
273 void *key;
274 void *mackey;
275 };
276
277 struct cryptop_data {
278 struct csession *cse;
279
280 char *buf;
281 char *obuf;
282 char *aad;
283 bool done;
284 };
285
286 struct fcrypt {
287 TAILQ_HEAD(csessionlist, csession) csessions;
288 int sesn;
289 struct mtx lock;
290 };
291
292 static bool use_outputbuffers;
293 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW,
294 &use_outputbuffers, 0,
295 "Use separate output buffers for /dev/crypto requests.");
296
297 static bool use_separate_aad;
298 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW,
299 &use_separate_aad, 0,
300 "Use separate AAD buffer for /dev/crypto requests.");
301
302 static MALLOC_DEFINE(M_CRYPTODEV, "cryptodev", "/dev/crypto data buffers");
303
304 /*
305 * Check a crypto identifier to see if it requested
306 * a software device/driver. This can be done either
307 * by device name/class or through search constraints.
308 */
309 static int
checkforsoftware(int * cridp)310 checkforsoftware(int *cridp)
311 {
312 int crid;
313
314 crid = *cridp;
315
316 if (!crypto_devallowsoft) {
317 if (crid & CRYPTOCAP_F_SOFTWARE) {
318 if (crid & CRYPTOCAP_F_HARDWARE) {
319 *cridp = CRYPTOCAP_F_HARDWARE;
320 return 0;
321 }
322 return EINVAL;
323 }
324 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 &&
325 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0)
326 return EINVAL;
327 }
328 return 0;
329 }
330
331 static int
cse_create(struct fcrypt * fcr,struct session2_op * sop)332 cse_create(struct fcrypt *fcr, struct session2_op *sop)
333 {
334 struct crypto_session_params csp;
335 struct csession *cse;
336 const struct enc_xform *txform;
337 const struct auth_hash *thash;
338 void *key = NULL;
339 void *mackey = NULL;
340 crypto_session_t cses;
341 int crid, error, mac;
342
343 mac = sop->mac;
344 #ifdef COMPAT_FREEBSD12
345 switch (sop->mac) {
346 case CRYPTO_AES_128_NIST_GMAC:
347 case CRYPTO_AES_192_NIST_GMAC:
348 case CRYPTO_AES_256_NIST_GMAC:
349 /* Should always be paired with GCM. */
350 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) {
351 CRYPTDEB("GMAC without GCM");
352 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
353 return (EINVAL);
354 }
355 if (sop->keylen != sop->mackeylen) {
356 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
357 return (EINVAL);
358 }
359 mac = 0;
360 break;
361 case CRYPTO_AES_CCM_CBC_MAC:
362 /* Should always be paired with CCM. */
363 if (sop->cipher != CRYPTO_AES_CCM_16) {
364 CRYPTDEB("CBC-MAC without CCM");
365 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
366 return (EINVAL);
367 }
368 if (sop->keylen != sop->mackeylen) {
369 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
370 return (EINVAL);
371 }
372 mac = 0;
373 break;
374 }
375 #endif
376
377 memset(&csp, 0, sizeof(csp));
378 if (use_outputbuffers)
379 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
380 if (mac != 0) {
381 csp.csp_auth_alg = mac;
382 csp.csp_auth_klen = sop->mackeylen;
383 }
384 if (sop->cipher != 0) {
385 csp.csp_cipher_alg = sop->cipher;
386 csp.csp_cipher_klen = sop->keylen;
387 }
388 thash = crypto_auth_hash(&csp);
389 txform = crypto_cipher(&csp);
390
391 if (txform != NULL && txform->macsize != 0) {
392 if (mac != 0) {
393 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
394 return (EINVAL);
395 }
396 csp.csp_mode = CSP_MODE_AEAD;
397 } else if (txform != NULL && thash != NULL) {
398 csp.csp_mode = CSP_MODE_ETA;
399 } else if (txform != NULL) {
400 csp.csp_mode = CSP_MODE_CIPHER;
401 } else if (thash != NULL) {
402 csp.csp_mode = CSP_MODE_DIGEST;
403 } else {
404 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
405 return (EINVAL);
406 }
407
408 switch (csp.csp_mode) {
409 case CSP_MODE_AEAD:
410 case CSP_MODE_ETA:
411 if (use_separate_aad)
412 csp.csp_flags |= CSP_F_SEPARATE_AAD;
413 break;
414 }
415
416 if (txform != NULL) {
417 if (sop->keylen > txform->maxkey ||
418 sop->keylen < txform->minkey) {
419 CRYPTDEB("invalid cipher parameters");
420 error = EINVAL;
421 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
422 goto bail;
423 }
424
425 key = malloc(csp.csp_cipher_klen, M_CRYPTODEV, M_WAITOK);
426 error = copyin(sop->key, key, csp.csp_cipher_klen);
427 if (error) {
428 CRYPTDEB("invalid key");
429 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
430 goto bail;
431 }
432 csp.csp_cipher_key = key;
433 csp.csp_ivlen = txform->ivsize;
434 }
435
436 if (thash != NULL) {
437 if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) {
438 CRYPTDEB("invalid mac key length");
439 error = EINVAL;
440 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
441 goto bail;
442 }
443
444 if (csp.csp_auth_klen != 0) {
445 mackey = malloc(csp.csp_auth_klen, M_CRYPTODEV,
446 M_WAITOK);
447 error = copyin(sop->mackey, mackey, csp.csp_auth_klen);
448 if (error) {
449 CRYPTDEB("invalid mac key");
450 SDT_PROBE1(opencrypto, dev, ioctl, error,
451 __LINE__);
452 goto bail;
453 }
454 csp.csp_auth_key = mackey;
455 }
456
457 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC)
458 csp.csp_ivlen = AES_GCM_IV_LEN;
459 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC)
460 csp.csp_ivlen = AES_CCM_IV_LEN;
461 }
462
463 if (sop->ivlen != 0) {
464 if (csp.csp_ivlen == 0) {
465 CRYPTDEB("does not support an IV");
466 error = EINVAL;
467 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
468 goto bail;
469 }
470 csp.csp_ivlen = sop->ivlen;
471 }
472 if (sop->maclen != 0) {
473 if (!(thash != NULL || csp.csp_mode == CSP_MODE_AEAD)) {
474 CRYPTDEB("does not support a MAC");
475 error = EINVAL;
476 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
477 goto bail;
478 }
479 csp.csp_auth_mlen = sop->maclen;
480 }
481
482 crid = sop->crid;
483 error = checkforsoftware(&crid);
484 if (error) {
485 CRYPTDEB("checkforsoftware");
486 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
487 goto bail;
488 }
489 error = crypto_newsession(&cses, &csp, crid);
490 if (error) {
491 CRYPTDEB("crypto_newsession");
492 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
493 goto bail;
494 }
495
496 cse = malloc(sizeof(struct csession), M_CRYPTODEV, M_WAITOK | M_ZERO);
497 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF);
498 refcount_init(&cse->refs, 1);
499 cse->key = key;
500 cse->mackey = mackey;
501 cse->cses = cses;
502 if (sop->maclen != 0)
503 cse->hashsize = sop->maclen;
504 else if (thash != NULL)
505 cse->hashsize = thash->hashsize;
506 else if (csp.csp_mode == CSP_MODE_AEAD)
507 cse->hashsize = txform->macsize;
508 cse->ivsize = csp.csp_ivlen;
509
510 /*
511 * NB: This isn't necessarily the block size of the underlying
512 * MAC or cipher but is instead a restriction on valid input
513 * sizes.
514 */
515 if (txform != NULL)
516 cse->blocksize = txform->blocksize;
517 else
518 cse->blocksize = 1;
519
520 mtx_lock(&fcr->lock);
521 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next);
522 cse->ses = fcr->sesn++;
523 mtx_unlock(&fcr->lock);
524
525 sop->ses = cse->ses;
526
527 /* return hardware/driver id */
528 sop->crid = crypto_ses2hid(cse->cses);
529 bail:
530 if (error) {
531 free(key, M_CRYPTODEV);
532 free(mackey, M_CRYPTODEV);
533 }
534 return (error);
535 }
536
537 static struct csession *
cse_find(struct fcrypt * fcr,u_int ses)538 cse_find(struct fcrypt *fcr, u_int ses)
539 {
540 struct csession *cse;
541
542 mtx_lock(&fcr->lock);
543 TAILQ_FOREACH(cse, &fcr->csessions, next) {
544 if (cse->ses == ses) {
545 refcount_acquire(&cse->refs);
546 mtx_unlock(&fcr->lock);
547 return (cse);
548 }
549 }
550 mtx_unlock(&fcr->lock);
551 return (NULL);
552 }
553
554 static void
cse_free(struct csession * cse)555 cse_free(struct csession *cse)
556 {
557
558 if (!refcount_release(&cse->refs))
559 return;
560 crypto_freesession(cse->cses);
561 mtx_destroy(&cse->lock);
562 if (cse->key)
563 free(cse->key, M_CRYPTODEV);
564 if (cse->mackey)
565 free(cse->mackey, M_CRYPTODEV);
566 free(cse, M_CRYPTODEV);
567 }
568
569 static bool
cse_delete(struct fcrypt * fcr,u_int ses)570 cse_delete(struct fcrypt *fcr, u_int ses)
571 {
572 struct csession *cse;
573
574 mtx_lock(&fcr->lock);
575 TAILQ_FOREACH(cse, &fcr->csessions, next) {
576 if (cse->ses == ses) {
577 TAILQ_REMOVE(&fcr->csessions, cse, next);
578 mtx_unlock(&fcr->lock);
579 cse_free(cse);
580 return (true);
581 }
582 }
583 mtx_unlock(&fcr->lock);
584 return (false);
585 }
586
587 static struct cryptop_data *
cod_alloc(struct csession * cse,size_t aad_len,size_t len)588 cod_alloc(struct csession *cse, size_t aad_len, size_t len)
589 {
590 struct cryptop_data *cod;
591
592 cod = malloc(sizeof(struct cryptop_data), M_CRYPTODEV, M_WAITOK |
593 M_ZERO);
594
595 cod->cse = cse;
596 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) {
597 if (aad_len != 0)
598 cod->aad = malloc(aad_len, M_CRYPTODEV, M_WAITOK);
599 cod->buf = malloc(len, M_CRYPTODEV, M_WAITOK);
600 } else
601 cod->buf = malloc(aad_len + len, M_CRYPTODEV, M_WAITOK);
602 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT)
603 cod->obuf = malloc(len, M_CRYPTODEV, M_WAITOK);
604 return (cod);
605 }
606
607 static void
cod_free(struct cryptop_data * cod)608 cod_free(struct cryptop_data *cod)
609 {
610
611 free(cod->aad, M_CRYPTODEV);
612 free(cod->obuf, M_CRYPTODEV);
613 free(cod->buf, M_CRYPTODEV);
614 free(cod, M_CRYPTODEV);
615 }
616
617 static int
cryptodev_cb(struct cryptop * crp)618 cryptodev_cb(struct cryptop *crp)
619 {
620 struct cryptop_data *cod = crp->crp_opaque;
621
622 /*
623 * Lock to ensure the wakeup() is not missed by the loops
624 * waiting on cod->done in cryptodev_op() and
625 * cryptodev_aead().
626 */
627 mtx_lock(&cod->cse->lock);
628 cod->done = true;
629 mtx_unlock(&cod->cse->lock);
630 wakeup(cod);
631 return (0);
632 }
633
634 static int
cryptodev_op(struct csession * cse,const struct crypt_op * cop)635 cryptodev_op(struct csession *cse, const struct crypt_op *cop)
636 {
637 const struct crypto_session_params *csp;
638 struct cryptop_data *cod = NULL;
639 struct cryptop *crp = NULL;
640 char *dst;
641 int error;
642
643 if (cop->len > 256*1024-4) {
644 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
645 return (E2BIG);
646 }
647
648 if ((cop->len % cse->blocksize) != 0) {
649 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
650 return (EINVAL);
651 }
652
653 if (cop->mac && cse->hashsize == 0) {
654 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
655 return (EINVAL);
656 }
657
658 /*
659 * The COP_F_CIPHER_FIRST flag predates explicit session
660 * modes, but the only way it was used was for EtA so allow it
661 * as long as it is consistent with EtA.
662 */
663 if (cop->flags & COP_F_CIPHER_FIRST) {
664 if (cop->op != COP_ENCRYPT) {
665 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
666 return (EINVAL);
667 }
668 }
669
670 cod = cod_alloc(cse, 0, cop->len + cse->hashsize);
671 dst = cop->dst;
672
673 crp = crypto_getreq(cse->cses, M_WAITOK);
674
675 error = copyin(cop->src, cod->buf, cop->len);
676 if (error) {
677 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
678 goto bail;
679 }
680 crp->crp_payload_start = 0;
681 crp->crp_payload_length = cop->len;
682 if (cse->hashsize)
683 crp->crp_digest_start = cop->len;
684
685 csp = crypto_get_params(cse->cses);
686 switch (csp->csp_mode) {
687 case CSP_MODE_COMPRESS:
688 switch (cop->op) {
689 case COP_ENCRYPT:
690 crp->crp_op = CRYPTO_OP_COMPRESS;
691 break;
692 case COP_DECRYPT:
693 crp->crp_op = CRYPTO_OP_DECOMPRESS;
694 break;
695 default:
696 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
697 error = EINVAL;
698 goto bail;
699 }
700 break;
701 case CSP_MODE_CIPHER:
702 if (cop->len == 0 ||
703 (cop->iv == NULL && cop->len == cse->ivsize)) {
704 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
705 error = EINVAL;
706 goto bail;
707 }
708 switch (cop->op) {
709 case COP_ENCRYPT:
710 crp->crp_op = CRYPTO_OP_ENCRYPT;
711 break;
712 case COP_DECRYPT:
713 crp->crp_op = CRYPTO_OP_DECRYPT;
714 break;
715 default:
716 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
717 error = EINVAL;
718 goto bail;
719 }
720 break;
721 case CSP_MODE_DIGEST:
722 switch (cop->op) {
723 case 0:
724 case COP_ENCRYPT:
725 case COP_DECRYPT:
726 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
727 if (cod->obuf != NULL)
728 crp->crp_digest_start = 0;
729 break;
730 default:
731 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
732 error = EINVAL;
733 goto bail;
734 }
735 break;
736 case CSP_MODE_AEAD:
737 if (cse->ivsize != 0 && cop->iv == NULL) {
738 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
739 error = EINVAL;
740 goto bail;
741 }
742 /* FALLTHROUGH */
743 case CSP_MODE_ETA:
744 switch (cop->op) {
745 case COP_ENCRYPT:
746 crp->crp_op = CRYPTO_OP_ENCRYPT |
747 CRYPTO_OP_COMPUTE_DIGEST;
748 break;
749 case COP_DECRYPT:
750 crp->crp_op = CRYPTO_OP_DECRYPT |
751 CRYPTO_OP_VERIFY_DIGEST;
752 break;
753 default:
754 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
755 error = EINVAL;
756 goto bail;
757 }
758 break;
759 default:
760 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
761 error = EINVAL;
762 goto bail;
763 }
764
765 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH);
766 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize);
767 if (cod->obuf)
768 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize);
769 crp->crp_callback = cryptodev_cb;
770 crp->crp_opaque = cod;
771
772 if (cop->iv) {
773 if (cse->ivsize == 0) {
774 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
775 error = EINVAL;
776 goto bail;
777 }
778 error = copyin(cop->iv, crp->crp_iv, cse->ivsize);
779 if (error) {
780 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
781 goto bail;
782 }
783 crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
784 } else if (cse->ivsize != 0) {
785 if (crp->crp_payload_length < cse->ivsize) {
786 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
787 error = EINVAL;
788 goto bail;
789 }
790 crp->crp_iv_start = 0;
791 crp->crp_payload_length -= cse->ivsize;
792 if (crp->crp_payload_length != 0)
793 crp->crp_payload_start = cse->ivsize;
794 dst += cse->ivsize;
795 }
796
797 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
798 error = copyin(cop->mac, cod->buf + crp->crp_digest_start,
799 cse->hashsize);
800 if (error) {
801 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
802 goto bail;
803 }
804 }
805 again:
806 /*
807 * Let the dispatch run unlocked, then, interlock against the
808 * callback before checking if the operation completed and going
809 * to sleep. This insures drivers don't inherit our lock which
810 * results in a lock order reversal between crypto_dispatch forced
811 * entry and the crypto_done callback into us.
812 */
813 error = crypto_dispatch(crp);
814 if (error != 0) {
815 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
816 goto bail;
817 }
818
819 mtx_lock(&cse->lock);
820 while (!cod->done)
821 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
822 mtx_unlock(&cse->lock);
823
824 if (crp->crp_etype == EAGAIN) {
825 crp->crp_etype = 0;
826 cod->done = false;
827 goto again;
828 }
829
830 if (crp->crp_etype != 0) {
831 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
832 error = crp->crp_etype;
833 goto bail;
834 }
835
836 if (cop->dst != NULL) {
837 error = copyout(cod->obuf != NULL ? cod->obuf :
838 cod->buf + crp->crp_payload_start, dst,
839 crp->crp_payload_length);
840 if (error) {
841 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
842 goto bail;
843 }
844 }
845
846 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) {
847 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) +
848 crp->crp_digest_start, cop->mac, cse->hashsize);
849 if (error) {
850 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
851 goto bail;
852 }
853 }
854
855 bail:
856 crypto_freereq(crp);
857 cod_free(cod);
858
859 return (error);
860 }
861
862 static int
cryptodev_aead(struct csession * cse,struct crypt_aead * caead)863 cryptodev_aead(struct csession *cse, struct crypt_aead *caead)
864 {
865 const struct crypto_session_params *csp;
866 struct cryptop_data *cod = NULL;
867 struct cryptop *crp = NULL;
868 char *dst;
869 int error;
870
871 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) {
872 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
873 return (E2BIG);
874 }
875
876 if ((caead->len % cse->blocksize) != 0) {
877 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
878 return (EINVAL);
879 }
880
881 if (cse->hashsize == 0 || caead->tag == NULL) {
882 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
883 return (EINVAL);
884 }
885
886 /*
887 * The COP_F_CIPHER_FIRST flag predates explicit session
888 * modes, but the only way it was used was for EtA so allow it
889 * as long as it is consistent with EtA.
890 */
891 if (caead->flags & COP_F_CIPHER_FIRST) {
892 if (caead->op != COP_ENCRYPT) {
893 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
894 return (EINVAL);
895 }
896 }
897
898 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize);
899 dst = caead->dst;
900
901 crp = crypto_getreq(cse->cses, M_WAITOK);
902
903 if (cod->aad != NULL)
904 error = copyin(caead->aad, cod->aad, caead->aadlen);
905 else
906 error = copyin(caead->aad, cod->buf, caead->aadlen);
907 if (error) {
908 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
909 goto bail;
910 }
911 crp->crp_aad = cod->aad;
912 crp->crp_aad_start = 0;
913 crp->crp_aad_length = caead->aadlen;
914
915 if (cod->aad != NULL)
916 crp->crp_payload_start = 0;
917 else
918 crp->crp_payload_start = caead->aadlen;
919 error = copyin(caead->src, cod->buf + crp->crp_payload_start,
920 caead->len);
921 if (error) {
922 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
923 goto bail;
924 }
925 crp->crp_payload_length = caead->len;
926 if (caead->op == COP_ENCRYPT && cod->obuf != NULL)
927 crp->crp_digest_start = crp->crp_payload_output_start +
928 caead->len;
929 else
930 crp->crp_digest_start = crp->crp_payload_start + caead->len;
931
932 csp = crypto_get_params(cse->cses);
933 switch (csp->csp_mode) {
934 case CSP_MODE_AEAD:
935 case CSP_MODE_ETA:
936 switch (caead->op) {
937 case COP_ENCRYPT:
938 crp->crp_op = CRYPTO_OP_ENCRYPT |
939 CRYPTO_OP_COMPUTE_DIGEST;
940 break;
941 case COP_DECRYPT:
942 crp->crp_op = CRYPTO_OP_DECRYPT |
943 CRYPTO_OP_VERIFY_DIGEST;
944 break;
945 default:
946 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
947 error = EINVAL;
948 goto bail;
949 }
950 break;
951 default:
952 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
953 error = EINVAL;
954 goto bail;
955 }
956
957 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH);
958 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len +
959 cse->hashsize);
960 if (cod->obuf != NULL)
961 crypto_use_output_buf(crp, cod->obuf, caead->len +
962 cse->hashsize);
963 crp->crp_callback = cryptodev_cb;
964 crp->crp_opaque = cod;
965
966 if (caead->iv) {
967 /*
968 * Permit a 16-byte IV for AES-XTS, but only use the
969 * first 8 bytes as a block number.
970 */
971 if (csp->csp_mode == CSP_MODE_ETA &&
972 csp->csp_cipher_alg == CRYPTO_AES_XTS &&
973 caead->ivlen == AES_BLOCK_LEN)
974 caead->ivlen = AES_XTS_IV_LEN;
975
976 if (cse->ivsize == 0) {
977 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
978 error = EINVAL;
979 goto bail;
980 }
981 if (caead->ivlen != cse->ivsize) {
982 error = EINVAL;
983 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
984 goto bail;
985 }
986
987 error = copyin(caead->iv, crp->crp_iv, cse->ivsize);
988 if (error) {
989 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
990 goto bail;
991 }
992 crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
993 } else {
994 error = EINVAL;
995 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
996 goto bail;
997 }
998
999 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
1000 error = copyin(caead->tag, cod->buf + crp->crp_digest_start,
1001 cse->hashsize);
1002 if (error) {
1003 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1004 goto bail;
1005 }
1006 }
1007 again:
1008 /*
1009 * Let the dispatch run unlocked, then, interlock against the
1010 * callback before checking if the operation completed and going
1011 * to sleep. This insures drivers don't inherit our lock which
1012 * results in a lock order reversal between crypto_dispatch forced
1013 * entry and the crypto_done callback into us.
1014 */
1015 error = crypto_dispatch(crp);
1016 if (error != 0) {
1017 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1018 goto bail;
1019 }
1020
1021 mtx_lock(&cse->lock);
1022 while (!cod->done)
1023 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
1024 mtx_unlock(&cse->lock);
1025
1026 if (crp->crp_etype == EAGAIN) {
1027 crp->crp_etype = 0;
1028 cod->done = false;
1029 goto again;
1030 }
1031
1032 if (crp->crp_etype != 0) {
1033 error = crp->crp_etype;
1034 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1035 goto bail;
1036 }
1037
1038 if (caead->dst != NULL) {
1039 error = copyout(cod->obuf != NULL ? cod->obuf :
1040 cod->buf + crp->crp_payload_start, dst,
1041 crp->crp_payload_length);
1042 if (error) {
1043 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1044 goto bail;
1045 }
1046 }
1047
1048 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) {
1049 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) +
1050 crp->crp_digest_start, caead->tag, cse->hashsize);
1051 if (error) {
1052 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1053 goto bail;
1054 }
1055 }
1056
1057 bail:
1058 crypto_freereq(crp);
1059 cod_free(cod);
1060
1061 return (error);
1062 }
1063
1064 static int
cryptodev_find(struct crypt_find_op * find)1065 cryptodev_find(struct crypt_find_op *find)
1066 {
1067 device_t dev;
1068 size_t fnlen = sizeof find->name;
1069
1070 if (find->crid != -1) {
1071 dev = crypto_find_device_byhid(find->crid);
1072 if (dev == NULL)
1073 return (ENOENT);
1074 strncpy(find->name, device_get_nameunit(dev), fnlen);
1075 find->name[fnlen - 1] = '\x0';
1076 } else {
1077 find->name[fnlen - 1] = '\x0';
1078 find->crid = crypto_find_driver(find->name);
1079 if (find->crid == -1)
1080 return (ENOENT);
1081 }
1082 return (0);
1083 }
1084
1085 static void
fcrypt_dtor(void * data)1086 fcrypt_dtor(void *data)
1087 {
1088 struct fcrypt *fcr = data;
1089 struct csession *cse;
1090
1091 while ((cse = TAILQ_FIRST(&fcr->csessions))) {
1092 TAILQ_REMOVE(&fcr->csessions, cse, next);
1093 KASSERT(refcount_load(&cse->refs) == 1,
1094 ("%s: crypto session %p with %d refs", __func__, cse,
1095 refcount_load(&cse->refs)));
1096 cse_free(cse);
1097 }
1098 mtx_destroy(&fcr->lock);
1099 free(fcr, M_CRYPTODEV);
1100 }
1101
1102 static int
crypto_open(struct cdev * dev,int oflags,int devtype,struct thread * td)1103 crypto_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
1104 {
1105 struct fcrypt *fcr;
1106 int error;
1107
1108 fcr = malloc(sizeof(struct fcrypt), M_CRYPTODEV, M_WAITOK | M_ZERO);
1109 TAILQ_INIT(&fcr->csessions);
1110 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF);
1111 error = devfs_set_cdevpriv(fcr, fcrypt_dtor);
1112 if (error)
1113 fcrypt_dtor(fcr);
1114 return (error);
1115 }
1116
1117 static int
crypto_ioctl(struct cdev * dev,u_long cmd,caddr_t data,int flag,struct thread * td)1118 crypto_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag,
1119 struct thread *td)
1120 {
1121 struct fcrypt *fcr;
1122 struct csession *cse;
1123 struct session2_op *sop;
1124 struct crypt_op *cop;
1125 struct crypt_aead *caead;
1126 uint32_t ses;
1127 int error = 0;
1128 union {
1129 struct session2_op sopc;
1130 #ifdef COMPAT_FREEBSD32
1131 struct crypt_op copc;
1132 struct crypt_aead aeadc;
1133 #endif
1134 } thunk;
1135 #ifdef COMPAT_FREEBSD32
1136 u_long cmd32;
1137 void *data32;
1138
1139 cmd32 = 0;
1140 data32 = NULL;
1141 switch (cmd) {
1142 case CIOCGSESSION32:
1143 cmd32 = cmd;
1144 data32 = data;
1145 cmd = CIOCGSESSION;
1146 data = (void *)&thunk.sopc;
1147 session_op_from_32((struct session_op32 *)data32, &thunk.sopc);
1148 break;
1149 case CIOCGSESSION232:
1150 cmd32 = cmd;
1151 data32 = data;
1152 cmd = CIOCGSESSION2;
1153 data = (void *)&thunk.sopc;
1154 session2_op_from_32((struct session2_op32 *)data32,
1155 &thunk.sopc);
1156 break;
1157 case CIOCCRYPT32:
1158 cmd32 = cmd;
1159 data32 = data;
1160 cmd = CIOCCRYPT;
1161 data = (void *)&thunk.copc;
1162 crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc);
1163 break;
1164 case CIOCCRYPTAEAD32:
1165 cmd32 = cmd;
1166 data32 = data;
1167 cmd = CIOCCRYPTAEAD;
1168 data = (void *)&thunk.aeadc;
1169 crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc);
1170 break;
1171 }
1172 #endif
1173
1174 devfs_get_cdevpriv((void **)&fcr);
1175
1176 switch (cmd) {
1177 #ifdef COMPAT_FREEBSD12
1178 case CRIOGET:
1179 /*
1180 * NB: This may fail in cases that the old
1181 * implementation did not if the current process has
1182 * restricted filesystem access (e.g. running in a
1183 * jail that does not expose /dev/crypto or in
1184 * capability mode).
1185 */
1186 error = kern_openat(td, AT_FDCWD, "/dev/crypto", UIO_SYSSPACE,
1187 O_RDWR, 0);
1188 if (error == 0)
1189 *(uint32_t *)data = td->td_retval[0];
1190 break;
1191 #endif
1192 case CIOCGSESSION:
1193 case CIOCGSESSION2:
1194 if (cmd == CIOCGSESSION) {
1195 session2_op_from_op((void *)data, &thunk.sopc);
1196 sop = &thunk.sopc;
1197 } else
1198 sop = (struct session2_op *)data;
1199
1200 error = cse_create(fcr, sop);
1201 if (cmd == CIOCGSESSION && error == 0)
1202 session2_op_to_op(sop, (void *)data);
1203 break;
1204 case CIOCFSESSION:
1205 ses = *(uint32_t *)data;
1206 if (!cse_delete(fcr, ses)) {
1207 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1208 return (EINVAL);
1209 }
1210 break;
1211 case CIOCCRYPT:
1212 cop = (struct crypt_op *)data;
1213 cse = cse_find(fcr, cop->ses);
1214 if (cse == NULL) {
1215 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1216 return (EINVAL);
1217 }
1218 error = cryptodev_op(cse, cop);
1219 cse_free(cse);
1220 break;
1221 case CIOCFINDDEV:
1222 error = cryptodev_find((struct crypt_find_op *)data);
1223 break;
1224 case CIOCCRYPTAEAD:
1225 caead = (struct crypt_aead *)data;
1226 cse = cse_find(fcr, caead->ses);
1227 if (cse == NULL) {
1228 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1229 return (EINVAL);
1230 }
1231 error = cryptodev_aead(cse, caead);
1232 cse_free(cse);
1233 break;
1234 default:
1235 error = EINVAL;
1236 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1237 break;
1238 }
1239
1240 #ifdef COMPAT_FREEBSD32
1241 switch (cmd32) {
1242 case CIOCGSESSION32:
1243 if (error == 0)
1244 session_op_to_32((void *)data, data32);
1245 break;
1246 case CIOCGSESSION232:
1247 if (error == 0)
1248 session2_op_to_32((void *)data, data32);
1249 break;
1250 case CIOCCRYPT32:
1251 if (error == 0)
1252 crypt_op_to_32((void *)data, data32);
1253 break;
1254 case CIOCCRYPTAEAD32:
1255 if (error == 0)
1256 crypt_aead_to_32((void *)data, data32);
1257 break;
1258 }
1259 #endif
1260 return (error);
1261 }
1262
1263 static struct cdevsw crypto_cdevsw = {
1264 .d_version = D_VERSION,
1265 .d_open = crypto_open,
1266 .d_ioctl = crypto_ioctl,
1267 .d_name = "crypto",
1268 };
1269 static struct cdev *crypto_dev;
1270
1271 /*
1272 * Initialization code, both for static and dynamic loading.
1273 */
1274 static int
cryptodev_modevent(module_t mod,int type,void * unused)1275 cryptodev_modevent(module_t mod, int type, void *unused)
1276 {
1277 switch (type) {
1278 case MOD_LOAD:
1279 if (bootverbose)
1280 printf("crypto: <crypto device>\n");
1281 crypto_dev = make_dev(&crypto_cdevsw, 0,
1282 UID_ROOT, GID_WHEEL, 0666,
1283 "crypto");
1284 return 0;
1285 case MOD_UNLOAD:
1286 /*XXX disallow if active sessions */
1287 destroy_dev(crypto_dev);
1288 return 0;
1289 }
1290 return EINVAL;
1291 }
1292
1293 static moduledata_t cryptodev_mod = {
1294 "cryptodev",
1295 cryptodev_modevent,
1296 0
1297 };
1298 MODULE_VERSION(cryptodev, 1);
1299 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
1300 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1);
1301 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1);
1302