xref: /freebsd/sys/opencrypto/cryptodev.c (revision f9fd7337f63698f33239c58c07bf430198235a22)
1 /*	$OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $	*/
2 
3 /*-
4  * Copyright (c) 2001 Theo de Raadt
5  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
6  * Copyright (c) 2014 The FreeBSD Foundation
7  * All rights reserved.
8  *
9  * Portions of this software were developed by John-Mark Gurney
10  * under sponsorship of the FreeBSD Foundation and
11  * Rubicon Communications, LLC (Netgate).
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  *
17  * 1. Redistributions of source code must retain the above copyright
18  *   notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *   notice, this list of conditions and the following disclaimer in the
21  *   documentation and/or other materials provided with the distribution.
22  * 3. The name of the author may not be used to endorse or promote products
23  *   derived from this software without specific prior written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
26  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
27  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
28  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
29  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
30  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
34  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35  *
36  * Effort sponsored in part by the Defense Advanced Research Projects
37  * Agency (DARPA) and Air Force Research Laboratory, Air Force
38  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
39  */
40 
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD$");
43 
44 #include <sys/param.h>
45 #include <sys/systm.h>
46 #include <sys/malloc.h>
47 #include <sys/mbuf.h>
48 #include <sys/lock.h>
49 #include <sys/mutex.h>
50 #include <sys/sysctl.h>
51 #include <sys/file.h>
52 #include <sys/filedesc.h>
53 #include <sys/errno.h>
54 #include <sys/random.h>
55 #include <sys/conf.h>
56 #include <sys/kernel.h>
57 #include <sys/module.h>
58 #include <sys/fcntl.h>
59 #include <sys/bus.h>
60 #include <sys/user.h>
61 #include <sys/sdt.h>
62 
63 #include <opencrypto/cryptodev.h>
64 #include <opencrypto/xform.h>
65 
66 SDT_PROVIDER_DECLARE(opencrypto);
67 
68 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/);
69 
70 #ifdef COMPAT_FREEBSD32
71 #include <sys/mount.h>
72 #include <compat/freebsd32/freebsd32.h>
73 
74 struct session_op32 {
75 	uint32_t	cipher;
76 	uint32_t	mac;
77 	uint32_t	keylen;
78 	uint32_t	key;
79 	int		mackeylen;
80 	uint32_t	mackey;
81 	uint32_t	ses;
82 };
83 
84 struct session2_op32 {
85 	uint32_t	cipher;
86 	uint32_t	mac;
87 	uint32_t	keylen;
88 	uint32_t	key;
89 	int		mackeylen;
90 	uint32_t	mackey;
91 	uint32_t	ses;
92 	int		crid;
93 	int		pad[4];
94 };
95 
96 struct crypt_op32 {
97 	uint32_t	ses;
98 	uint16_t	op;
99 	uint16_t	flags;
100 	u_int		len;
101 	uint32_t	src, dst;
102 	uint32_t	mac;
103 	uint32_t	iv;
104 };
105 
106 struct crypt_aead32 {
107 	uint32_t	ses;
108 	uint16_t	op;
109 	uint16_t	flags;
110 	u_int		len;
111 	u_int		aadlen;
112 	u_int		ivlen;
113 	uint32_t	src;
114 	uint32_t	dst;
115 	uint32_t	aad;
116 	uint32_t	tag;
117 	uint32_t	iv;
118 };
119 
120 struct crparam32 {
121 	uint32_t	crp_p;
122 	u_int		crp_nbits;
123 };
124 
125 struct crypt_kop32 {
126 	u_int		crk_op;
127 	u_int		crk_status;
128 	u_short		crk_iparams;
129 	u_short		crk_oparams;
130 	u_int		crk_crid;
131 	struct crparam32	crk_param[CRK_MAXPARAM];
132 };
133 
134 #define	CIOCGSESSION32	_IOWR('c', 101, struct session_op32)
135 #define	CIOCCRYPT32	_IOWR('c', 103, struct crypt_op32)
136 #define	CIOCKEY32	_IOWR('c', 104, struct crypt_kop32)
137 #define	CIOCGSESSION232	_IOWR('c', 106, struct session2_op32)
138 #define	CIOCKEY232	_IOWR('c', 107, struct crypt_kop32)
139 #define	CIOCCRYPTAEAD32	_IOWR('c', 109, struct crypt_aead32)
140 
141 static void
142 session_op_from_32(const struct session_op32 *from, struct session2_op *to)
143 {
144 
145 	memset(to, 0, sizeof(*to));
146 	CP(*from, *to, cipher);
147 	CP(*from, *to, mac);
148 	CP(*from, *to, keylen);
149 	PTRIN_CP(*from, *to, key);
150 	CP(*from, *to, mackeylen);
151 	PTRIN_CP(*from, *to, mackey);
152 	CP(*from, *to, ses);
153 	to->crid = CRYPTOCAP_F_HARDWARE;
154 }
155 
156 static void
157 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to)
158 {
159 
160 	session_op_from_32((const struct session_op32 *)from, to);
161 	CP(*from, *to, crid);
162 }
163 
164 static void
165 session_op_to_32(const struct session2_op *from, struct session_op32 *to)
166 {
167 
168 	CP(*from, *to, cipher);
169 	CP(*from, *to, mac);
170 	CP(*from, *to, keylen);
171 	PTROUT_CP(*from, *to, key);
172 	CP(*from, *to, mackeylen);
173 	PTROUT_CP(*from, *to, mackey);
174 	CP(*from, *to, ses);
175 }
176 
177 static void
178 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to)
179 {
180 
181 	session_op_to_32(from, (struct session_op32 *)to);
182 	CP(*from, *to, crid);
183 }
184 
185 static void
186 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to)
187 {
188 
189 	CP(*from, *to, ses);
190 	CP(*from, *to, op);
191 	CP(*from, *to, flags);
192 	CP(*from, *to, len);
193 	PTRIN_CP(*from, *to, src);
194 	PTRIN_CP(*from, *to, dst);
195 	PTRIN_CP(*from, *to, mac);
196 	PTRIN_CP(*from, *to, iv);
197 }
198 
199 static void
200 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to)
201 {
202 
203 	CP(*from, *to, ses);
204 	CP(*from, *to, op);
205 	CP(*from, *to, flags);
206 	CP(*from, *to, len);
207 	PTROUT_CP(*from, *to, src);
208 	PTROUT_CP(*from, *to, dst);
209 	PTROUT_CP(*from, *to, mac);
210 	PTROUT_CP(*from, *to, iv);
211 }
212 
213 static void
214 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to)
215 {
216 
217 	CP(*from, *to, ses);
218 	CP(*from, *to, op);
219 	CP(*from, *to, flags);
220 	CP(*from, *to, len);
221 	CP(*from, *to, aadlen);
222 	CP(*from, *to, ivlen);
223 	PTRIN_CP(*from, *to, src);
224 	PTRIN_CP(*from, *to, dst);
225 	PTRIN_CP(*from, *to, aad);
226 	PTRIN_CP(*from, *to, tag);
227 	PTRIN_CP(*from, *to, iv);
228 }
229 
230 static void
231 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to)
232 {
233 
234 	CP(*from, *to, ses);
235 	CP(*from, *to, op);
236 	CP(*from, *to, flags);
237 	CP(*from, *to, len);
238 	CP(*from, *to, aadlen);
239 	CP(*from, *to, ivlen);
240 	PTROUT_CP(*from, *to, src);
241 	PTROUT_CP(*from, *to, dst);
242 	PTROUT_CP(*from, *to, aad);
243 	PTROUT_CP(*from, *to, tag);
244 	PTROUT_CP(*from, *to, iv);
245 }
246 
247 static void
248 crparam_from_32(const struct crparam32 *from, struct crparam *to)
249 {
250 
251 	PTRIN_CP(*from, *to, crp_p);
252 	CP(*from, *to, crp_nbits);
253 }
254 
255 static void
256 crparam_to_32(const struct crparam *from, struct crparam32 *to)
257 {
258 
259 	PTROUT_CP(*from, *to, crp_p);
260 	CP(*from, *to, crp_nbits);
261 }
262 
263 static void
264 crypt_kop_from_32(const struct crypt_kop32 *from, struct crypt_kop *to)
265 {
266 	int i;
267 
268 	CP(*from, *to, crk_op);
269 	CP(*from, *to, crk_status);
270 	CP(*from, *to, crk_iparams);
271 	CP(*from, *to, crk_oparams);
272 	CP(*from, *to, crk_crid);
273 	for (i = 0; i < CRK_MAXPARAM; i++)
274 		crparam_from_32(&from->crk_param[i], &to->crk_param[i]);
275 }
276 
277 static void
278 crypt_kop_to_32(const struct crypt_kop *from, struct crypt_kop32 *to)
279 {
280 	int i;
281 
282 	CP(*from, *to, crk_op);
283 	CP(*from, *to, crk_status);
284 	CP(*from, *to, crk_iparams);
285 	CP(*from, *to, crk_oparams);
286 	CP(*from, *to, crk_crid);
287 	for (i = 0; i < CRK_MAXPARAM; i++)
288 		crparam_to_32(&from->crk_param[i], &to->crk_param[i]);
289 }
290 #endif
291 
292 static void
293 session2_op_from_op(const struct session_op *from, struct session2_op *to)
294 {
295 
296 	memset(to, 0, sizeof(*to));
297 	memcpy(to, from, sizeof(*from));
298 	to->crid = CRYPTOCAP_F_HARDWARE;
299 }
300 
301 static void
302 session2_op_to_op(const struct session2_op *from, struct session_op *to)
303 {
304 
305 	memcpy(to, from, sizeof(*to));
306 }
307 
308 struct csession {
309 	TAILQ_ENTRY(csession) next;
310 	crypto_session_t cses;
311 	volatile u_int	refs;
312 	uint32_t	ses;
313 	struct mtx	lock;		/* for op submission */
314 
315 	struct enc_xform *txform;
316 	int		hashsize;
317 	int		ivsize;
318 	int		mode;
319 
320 	void		*key;
321 	void		*mackey;
322 };
323 
324 struct cryptop_data {
325 	struct csession *cse;
326 
327 	char		*buf;
328 	char		*obuf;
329 	char		*aad;
330 	bool		done;
331 };
332 
333 struct fcrypt {
334 	TAILQ_HEAD(csessionlist, csession) csessions;
335 	int		sesn;
336 	struct mtx	lock;
337 };
338 
339 static bool use_outputbuffers;
340 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW,
341     &use_outputbuffers, 0,
342     "Use separate output buffers for /dev/crypto requests.");
343 
344 static bool use_separate_aad;
345 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW,
346     &use_separate_aad, 0,
347     "Use separate AAD buffer for /dev/crypto requests.");
348 
349 static struct timeval warninterval = { .tv_sec = 60, .tv_usec = 0 };
350 SYSCTL_TIMEVAL_SEC(_kern, OID_AUTO, cryptodev_warn_interval, CTLFLAG_RW,
351     &warninterval,
352     "Delay in seconds between warnings of deprecated /dev/crypto algorithms");
353 
354 static int cryptof_ioctl(struct file *, u_long, void *, struct ucred *,
355     struct thread *);
356 static int cryptof_stat(struct file *, struct stat *, struct ucred *,
357     struct thread *);
358 static int cryptof_close(struct file *, struct thread *);
359 static int cryptof_fill_kinfo(struct file *, struct kinfo_file *,
360     struct filedesc *);
361 
362 static struct fileops cryptofops = {
363     .fo_read = invfo_rdwr,
364     .fo_write = invfo_rdwr,
365     .fo_truncate = invfo_truncate,
366     .fo_ioctl = cryptof_ioctl,
367     .fo_poll = invfo_poll,
368     .fo_kqfilter = invfo_kqfilter,
369     .fo_stat = cryptof_stat,
370     .fo_close = cryptof_close,
371     .fo_chmod = invfo_chmod,
372     .fo_chown = invfo_chown,
373     .fo_sendfile = invfo_sendfile,
374     .fo_fill_kinfo = cryptof_fill_kinfo,
375 };
376 
377 /*
378  * Check a crypto identifier to see if it requested
379  * a software device/driver.  This can be done either
380  * by device name/class or through search constraints.
381  */
382 static int
383 checkforsoftware(int *cridp)
384 {
385 	int crid;
386 
387 	crid = *cridp;
388 
389 	if (!crypto_devallowsoft) {
390 		if (crid & CRYPTOCAP_F_SOFTWARE) {
391 			if (crid & CRYPTOCAP_F_HARDWARE) {
392 				*cridp = CRYPTOCAP_F_HARDWARE;
393 				return 0;
394 			}
395 			return EINVAL;
396 		}
397 		if ((crid & CRYPTOCAP_F_HARDWARE) == 0 &&
398 		    (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0)
399 			return EINVAL;
400 	}
401 	return 0;
402 }
403 
404 static int
405 cse_create(struct fcrypt *fcr, struct session2_op *sop)
406 {
407 	struct crypto_session_params csp;
408 	struct csession *cse;
409 	struct enc_xform *txform;
410 	struct auth_hash *thash;
411 	void *key = NULL;
412 	void *mackey = NULL;
413 	crypto_session_t cses;
414 	int crid, error;
415 
416 	switch (sop->cipher) {
417 	case 0:
418 		txform = NULL;
419 		break;
420 	case CRYPTO_AES_CBC:
421 		txform = &enc_xform_rijndael128;
422 		break;
423 	case CRYPTO_AES_XTS:
424 		txform = &enc_xform_aes_xts;
425 		break;
426 	case CRYPTO_NULL_CBC:
427 		txform = &enc_xform_null;
428 		break;
429 	case CRYPTO_CAMELLIA_CBC:
430 		txform = &enc_xform_camellia;
431 		break;
432 	case CRYPTO_AES_ICM:
433 		txform = &enc_xform_aes_icm;
434 		break;
435 	case CRYPTO_AES_NIST_GCM_16:
436 		txform = &enc_xform_aes_nist_gcm;
437 		break;
438 	case CRYPTO_CHACHA20:
439 		txform = &enc_xform_chacha20;
440 		break;
441 	case CRYPTO_AES_CCM_16:
442 		txform = &enc_xform_ccm;
443 		break;
444 	default:
445 		CRYPTDEB("invalid cipher");
446 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
447 		return (EINVAL);
448 	}
449 
450 	switch (sop->mac) {
451 	case 0:
452 		thash = NULL;
453 		break;
454 	case CRYPTO_POLY1305:
455 		thash = &auth_hash_poly1305;
456 		break;
457 	case CRYPTO_SHA1_HMAC:
458 		thash = &auth_hash_hmac_sha1;
459 		break;
460 	case CRYPTO_SHA2_224_HMAC:
461 		thash = &auth_hash_hmac_sha2_224;
462 		break;
463 	case CRYPTO_SHA2_256_HMAC:
464 		thash = &auth_hash_hmac_sha2_256;
465 		break;
466 	case CRYPTO_SHA2_384_HMAC:
467 		thash = &auth_hash_hmac_sha2_384;
468 		break;
469 	case CRYPTO_SHA2_512_HMAC:
470 		thash = &auth_hash_hmac_sha2_512;
471 		break;
472 	case CRYPTO_RIPEMD160_HMAC:
473 		thash = &auth_hash_hmac_ripemd_160;
474 		break;
475 #ifdef COMPAT_FREEBSD12
476 	case CRYPTO_AES_128_NIST_GMAC:
477 	case CRYPTO_AES_192_NIST_GMAC:
478 	case CRYPTO_AES_256_NIST_GMAC:
479 		/* Should always be paired with GCM. */
480 		if (sop->cipher != CRYPTO_AES_NIST_GCM_16) {
481 			CRYPTDEB("GMAC without GCM");
482 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
483 			return (EINVAL);
484 		}
485 		break;
486 #endif
487 	case CRYPTO_AES_NIST_GMAC:
488 		switch (sop->mackeylen * 8) {
489 		case 128:
490 			thash = &auth_hash_nist_gmac_aes_128;
491 			break;
492 		case 192:
493 			thash = &auth_hash_nist_gmac_aes_192;
494 			break;
495 		case 256:
496 			thash = &auth_hash_nist_gmac_aes_256;
497 			break;
498 		default:
499 			CRYPTDEB("invalid GMAC key length");
500 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
501 			return (EINVAL);
502 		}
503 		break;
504 	case CRYPTO_AES_CCM_CBC_MAC:
505 		switch (sop->mackeylen) {
506 		case 16:
507 			thash = &auth_hash_ccm_cbc_mac_128;
508 			break;
509 		case 24:
510 			thash = &auth_hash_ccm_cbc_mac_192;
511 			break;
512 		case 32:
513 			thash = &auth_hash_ccm_cbc_mac_256;
514 			break;
515 		default:
516 			CRYPTDEB("Invalid CBC MAC key size %d", sop->keylen);
517 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
518 			return (EINVAL);
519 		}
520 		break;
521 	case CRYPTO_SHA1:
522 		thash = &auth_hash_sha1;
523 		break;
524 	case CRYPTO_SHA2_224:
525 		thash = &auth_hash_sha2_224;
526 		break;
527 	case CRYPTO_SHA2_256:
528 		thash = &auth_hash_sha2_256;
529 		break;
530 	case CRYPTO_SHA2_384:
531 		thash = &auth_hash_sha2_384;
532 		break;
533 	case CRYPTO_SHA2_512:
534 		thash = &auth_hash_sha2_512;
535 		break;
536 
537 	case CRYPTO_NULL_HMAC:
538 		thash = &auth_hash_null;
539 		break;
540 
541 	case CRYPTO_BLAKE2B:
542 		thash = &auth_hash_blake2b;
543 		break;
544 	case CRYPTO_BLAKE2S:
545 		thash = &auth_hash_blake2s;
546 		break;
547 
548 	default:
549 		CRYPTDEB("invalid mac");
550 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
551 		return (EINVAL);
552 	}
553 
554 	if (txform == NULL && thash == NULL) {
555 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
556 		return (EINVAL);
557 	}
558 
559 	memset(&csp, 0, sizeof(csp));
560 	if (use_outputbuffers)
561 		csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
562 
563 	if (sop->cipher == CRYPTO_AES_NIST_GCM_16) {
564 		switch (sop->mac) {
565 #ifdef COMPAT_FREEBSD12
566 		case CRYPTO_AES_128_NIST_GMAC:
567 		case CRYPTO_AES_192_NIST_GMAC:
568 		case CRYPTO_AES_256_NIST_GMAC:
569 			if (sop->keylen != sop->mackeylen) {
570 				SDT_PROBE1(opencrypto, dev, ioctl, error,
571 				    __LINE__);
572 				return (EINVAL);
573 			}
574 			break;
575 #endif
576 		case 0:
577 			break;
578 		default:
579 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
580 			return (EINVAL);
581 		}
582 		csp.csp_mode = CSP_MODE_AEAD;
583 	} else if (sop->cipher == CRYPTO_AES_CCM_16) {
584 		switch (sop->mac) {
585 #ifdef COMPAT_FREEBSD12
586 		case CRYPTO_AES_CCM_CBC_MAC:
587 			if (sop->keylen != sop->mackeylen) {
588 				SDT_PROBE1(opencrypto, dev, ioctl, error,
589 				    __LINE__);
590 				return (EINVAL);
591 			}
592 			thash = NULL;
593 			break;
594 #endif
595 		case 0:
596 			break;
597 		default:
598 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
599 			return (EINVAL);
600 		}
601 		csp.csp_mode = CSP_MODE_AEAD;
602 	} else if (txform != NULL && thash != NULL)
603 		csp.csp_mode = CSP_MODE_ETA;
604 	else if (txform != NULL)
605 		csp.csp_mode = CSP_MODE_CIPHER;
606 	else
607 		csp.csp_mode = CSP_MODE_DIGEST;
608 
609 	switch (csp.csp_mode) {
610 	case CSP_MODE_AEAD:
611 	case CSP_MODE_ETA:
612 		if (use_separate_aad)
613 			csp.csp_flags |= CSP_F_SEPARATE_AAD;
614 		break;
615 	}
616 
617 	if (txform != NULL) {
618 		csp.csp_cipher_alg = txform->type;
619 		csp.csp_cipher_klen = sop->keylen;
620 		if (sop->keylen > txform->maxkey ||
621 		    sop->keylen < txform->minkey) {
622 			CRYPTDEB("invalid cipher parameters");
623 			error = EINVAL;
624 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
625 			goto bail;
626 		}
627 
628 		key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK);
629 		error = copyin(sop->key, key, csp.csp_cipher_klen);
630 		if (error) {
631 			CRYPTDEB("invalid key");
632 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
633 			goto bail;
634 		}
635 		csp.csp_cipher_key = key;
636 		csp.csp_ivlen = txform->ivsize;
637 	}
638 
639 	if (thash != NULL) {
640 		csp.csp_auth_alg = thash->type;
641 		csp.csp_auth_klen = sop->mackeylen;
642 		if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) {
643 			CRYPTDEB("invalid mac key length");
644 			error = EINVAL;
645 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
646 			goto bail;
647 		}
648 
649 		if (csp.csp_auth_klen != 0) {
650 			mackey = malloc(csp.csp_auth_klen, M_XDATA, M_WAITOK);
651 			error = copyin(sop->mackey, mackey, csp.csp_auth_klen);
652 			if (error) {
653 				CRYPTDEB("invalid mac key");
654 				SDT_PROBE1(opencrypto, dev, ioctl, error,
655 				    __LINE__);
656 				goto bail;
657 			}
658 			csp.csp_auth_key = mackey;
659 		}
660 
661 		if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC)
662 			csp.csp_ivlen = AES_GCM_IV_LEN;
663 		if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC)
664 			csp.csp_ivlen = AES_CCM_IV_LEN;
665 	}
666 
667 	crid = sop->crid;
668 	error = checkforsoftware(&crid);
669 	if (error) {
670 		CRYPTDEB("checkforsoftware");
671 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
672 		goto bail;
673 	}
674 	error = crypto_newsession(&cses, &csp, crid);
675 	if (error) {
676 		CRYPTDEB("crypto_newsession");
677 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
678 		goto bail;
679 	}
680 
681 	cse = malloc(sizeof(struct csession), M_XDATA, M_WAITOK | M_ZERO);
682 	mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF);
683 	refcount_init(&cse->refs, 1);
684 	cse->key = key;
685 	cse->mackey = mackey;
686 	cse->mode = csp.csp_mode;
687 	cse->cses = cses;
688 	cse->txform = txform;
689 	if (thash != NULL)
690 		cse->hashsize = thash->hashsize;
691 	else if (csp.csp_cipher_alg == CRYPTO_AES_NIST_GCM_16)
692 		cse->hashsize = AES_GMAC_HASH_LEN;
693 	else if (csp.csp_cipher_alg == CRYPTO_AES_CCM_16)
694 		cse->hashsize = AES_CBC_MAC_HASH_LEN;
695 	cse->ivsize = csp.csp_ivlen;
696 
697 	mtx_lock(&fcr->lock);
698 	TAILQ_INSERT_TAIL(&fcr->csessions, cse, next);
699 	cse->ses = fcr->sesn++;
700 	mtx_unlock(&fcr->lock);
701 
702 	sop->ses = cse->ses;
703 
704 	/* return hardware/driver id */
705 	sop->crid = crypto_ses2hid(cse->cses);
706 bail:
707 	if (error) {
708 		free(key, M_XDATA);
709 		free(mackey, M_XDATA);
710 	}
711 	return (error);
712 }
713 
714 static struct csession *
715 cse_find(struct fcrypt *fcr, u_int ses)
716 {
717 	struct csession *cse;
718 
719 	mtx_lock(&fcr->lock);
720 	TAILQ_FOREACH(cse, &fcr->csessions, next) {
721 		if (cse->ses == ses) {
722 			refcount_acquire(&cse->refs);
723 			mtx_unlock(&fcr->lock);
724 			return (cse);
725 		}
726 	}
727 	mtx_unlock(&fcr->lock);
728 	return (NULL);
729 }
730 
731 static void
732 cse_free(struct csession *cse)
733 {
734 
735 	if (!refcount_release(&cse->refs))
736 		return;
737 	crypto_freesession(cse->cses);
738 	mtx_destroy(&cse->lock);
739 	if (cse->key)
740 		free(cse->key, M_XDATA);
741 	if (cse->mackey)
742 		free(cse->mackey, M_XDATA);
743 	free(cse, M_XDATA);
744 }
745 
746 static bool
747 cse_delete(struct fcrypt *fcr, u_int ses)
748 {
749 	struct csession *cse;
750 
751 	mtx_lock(&fcr->lock);
752 	TAILQ_FOREACH(cse, &fcr->csessions, next) {
753 		if (cse->ses == ses) {
754 			TAILQ_REMOVE(&fcr->csessions, cse, next);
755 			mtx_unlock(&fcr->lock);
756 			cse_free(cse);
757 			return (true);
758 		}
759 	}
760 	mtx_unlock(&fcr->lock);
761 	return (false);
762 }
763 
764 static struct cryptop_data *
765 cod_alloc(struct csession *cse, size_t aad_len, size_t len, struct thread *td)
766 {
767 	struct cryptop_data *cod;
768 
769 	cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO);
770 
771 	cod->cse = cse;
772 	if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) {
773 		if (aad_len != 0)
774 			cod->aad = malloc(aad_len, M_XDATA, M_WAITOK);
775 		cod->buf = malloc(len, M_XDATA, M_WAITOK);
776 	} else
777 		cod->buf = malloc(aad_len + len, M_XDATA, M_WAITOK);
778 	if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT)
779 		cod->obuf = malloc(len, M_XDATA, M_WAITOK);
780 	return (cod);
781 }
782 
783 static void
784 cod_free(struct cryptop_data *cod)
785 {
786 
787 	free(cod->aad, M_XDATA);
788 	free(cod->obuf, M_XDATA);
789 	free(cod->buf, M_XDATA);
790 	free(cod, M_XDATA);
791 }
792 
793 static int
794 cryptodev_cb(struct cryptop *crp)
795 {
796 	struct cryptop_data *cod = crp->crp_opaque;
797 
798 	/*
799 	 * Lock to ensure the wakeup() is not missed by the loops
800 	 * waiting on cod->done in cryptodev_op() and
801 	 * cryptodev_aead().
802 	 */
803 	mtx_lock(&cod->cse->lock);
804 	cod->done = true;
805 	mtx_unlock(&cod->cse->lock);
806 	wakeup(cod);
807 	return (0);
808 }
809 
810 static int
811 cryptodev_op(struct csession *cse, const struct crypt_op *cop,
812     struct ucred *active_cred, struct thread *td)
813 {
814 	struct cryptop_data *cod = NULL;
815 	struct cryptop *crp = NULL;
816 	char *dst;
817 	int error;
818 
819 	if (cop->len > 256*1024-4) {
820 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
821 		return (E2BIG);
822 	}
823 
824 	if (cse->txform) {
825 		if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) {
826 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
827 			return (EINVAL);
828 		}
829 	}
830 
831 	if (cop->mac && cse->hashsize == 0) {
832 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
833 		return (EINVAL);
834 	}
835 
836 	/*
837 	 * The COP_F_CIPHER_FIRST flag predates explicit session
838 	 * modes, but the only way it was used was for EtA so allow it
839 	 * as long as it is consistent with EtA.
840 	 */
841 	if (cop->flags & COP_F_CIPHER_FIRST) {
842 		if (cop->op != COP_ENCRYPT) {
843 			SDT_PROBE1(opencrypto, dev, ioctl, error,  __LINE__);
844 			return (EINVAL);
845 		}
846 	}
847 
848 	cod = cod_alloc(cse, 0, cop->len + cse->hashsize, td);
849 	dst = cop->dst;
850 
851 	crp = crypto_getreq(cse->cses, M_WAITOK);
852 
853 	error = copyin(cop->src, cod->buf, cop->len);
854 	if (error) {
855 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
856 		goto bail;
857 	}
858 	crp->crp_payload_start = 0;
859 	crp->crp_payload_length = cop->len;
860 	if (cse->hashsize)
861 		crp->crp_digest_start = cop->len;
862 
863 	switch (cse->mode) {
864 	case CSP_MODE_COMPRESS:
865 		switch (cop->op) {
866 		case COP_ENCRYPT:
867 			crp->crp_op = CRYPTO_OP_COMPRESS;
868 			break;
869 		case COP_DECRYPT:
870 			crp->crp_op = CRYPTO_OP_DECOMPRESS;
871 			break;
872 		default:
873 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
874 			error = EINVAL;
875 			goto bail;
876 		}
877 		break;
878 	case CSP_MODE_CIPHER:
879 		switch (cop->op) {
880 		case COP_ENCRYPT:
881 			crp->crp_op = CRYPTO_OP_ENCRYPT;
882 			break;
883 		case COP_DECRYPT:
884 			crp->crp_op = CRYPTO_OP_DECRYPT;
885 			break;
886 		default:
887 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
888 			error = EINVAL;
889 			goto bail;
890 		}
891 		break;
892 	case CSP_MODE_DIGEST:
893 		switch (cop->op) {
894 		case 0:
895 		case COP_ENCRYPT:
896 		case COP_DECRYPT:
897 			crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
898 			if (cod->obuf != NULL)
899 				crp->crp_digest_start = 0;
900 			break;
901 		default:
902 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
903 			error = EINVAL;
904 			goto bail;
905 		}
906 		break;
907 	case CSP_MODE_ETA:
908 		switch (cop->op) {
909 		case COP_ENCRYPT:
910 			crp->crp_op = CRYPTO_OP_ENCRYPT |
911 			    CRYPTO_OP_COMPUTE_DIGEST;
912 			break;
913 		case COP_DECRYPT:
914 			crp->crp_op = CRYPTO_OP_DECRYPT |
915 			    CRYPTO_OP_VERIFY_DIGEST;
916 			break;
917 		default:
918 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
919 			error = EINVAL;
920 			goto bail;
921 		}
922 		break;
923 	default:
924 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
925 		error = EINVAL;
926 		goto bail;
927 	}
928 
929 	crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH);
930 	crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize);
931 	if (cod->obuf)
932 		crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize);
933 	crp->crp_callback = cryptodev_cb;
934 	crp->crp_opaque = cod;
935 
936 	if (cop->iv) {
937 		if (cse->ivsize == 0) {
938 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
939 			error = EINVAL;
940 			goto bail;
941 		}
942 		error = copyin(cop->iv, crp->crp_iv, cse->ivsize);
943 		if (error) {
944 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
945 			goto bail;
946 		}
947 		crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
948 	} else if (cse->ivsize != 0) {
949 		crp->crp_iv_start = 0;
950 		crp->crp_payload_start += cse->ivsize;
951 		crp->crp_payload_length -= cse->ivsize;
952 		dst += cse->ivsize;
953 	}
954 
955 	if (cop->mac != NULL && crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
956 		error = copyin(cop->mac, cod->buf + crp->crp_digest_start,
957 		    cse->hashsize);
958 		if (error) {
959 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
960 			goto bail;
961 		}
962 	}
963 again:
964 	/*
965 	 * Let the dispatch run unlocked, then, interlock against the
966 	 * callback before checking if the operation completed and going
967 	 * to sleep.  This insures drivers don't inherit our lock which
968 	 * results in a lock order reversal between crypto_dispatch forced
969 	 * entry and the crypto_done callback into us.
970 	 */
971 	error = crypto_dispatch(crp);
972 	if (error != 0) {
973 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
974 		goto bail;
975 	}
976 
977 	mtx_lock(&cse->lock);
978 	while (!cod->done)
979 		mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
980 	mtx_unlock(&cse->lock);
981 
982 	if (crp->crp_etype == EAGAIN) {
983 		crp->crp_etype = 0;
984 		crp->crp_flags &= ~CRYPTO_F_DONE;
985 		cod->done = false;
986 		goto again;
987 	}
988 
989 	if (crp->crp_etype != 0) {
990 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
991 		error = crp->crp_etype;
992 		goto bail;
993 	}
994 
995 	if (cop->dst != NULL) {
996 		error = copyout(cod->obuf != NULL ? cod->obuf :
997 		    cod->buf + crp->crp_payload_start, dst,
998 		    crp->crp_payload_length);
999 		if (error) {
1000 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1001 			goto bail;
1002 		}
1003 	}
1004 
1005 	if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) {
1006 		error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) +
1007 		    crp->crp_digest_start, cop->mac, cse->hashsize);
1008 		if (error) {
1009 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1010 			goto bail;
1011 		}
1012 	}
1013 
1014 bail:
1015 	crypto_freereq(crp);
1016 	cod_free(cod);
1017 
1018 	return (error);
1019 }
1020 
1021 static int
1022 cryptodev_aead(struct csession *cse, struct crypt_aead *caead,
1023     struct ucred *active_cred, struct thread *td)
1024 {
1025 	struct cryptop_data *cod = NULL;
1026 	struct cryptop *crp = NULL;
1027 	char *dst;
1028 	int error;
1029 
1030 	if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) {
1031 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1032 		return (E2BIG);
1033 	}
1034 
1035 	if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL ||
1036 	    (caead->len % cse->txform->blocksize) != 0) {
1037 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1038 		return (EINVAL);
1039 	}
1040 
1041 	/*
1042 	 * The COP_F_CIPHER_FIRST flag predates explicit session
1043 	 * modes, but the only way it was used was for EtA so allow it
1044 	 * as long as it is consistent with EtA.
1045 	 */
1046 	if (caead->flags & COP_F_CIPHER_FIRST) {
1047 		if (caead->op != COP_ENCRYPT) {
1048 			SDT_PROBE1(opencrypto, dev, ioctl, error,  __LINE__);
1049 			return (EINVAL);
1050 		}
1051 	}
1052 
1053 	cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize, td);
1054 	dst = caead->dst;
1055 
1056 	crp = crypto_getreq(cse->cses, M_WAITOK);
1057 
1058 	if (cod->aad != NULL)
1059 		error = copyin(caead->aad, cod->aad, caead->aadlen);
1060 	else
1061 		error = copyin(caead->aad, cod->buf, caead->aadlen);
1062 	if (error) {
1063 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1064 		goto bail;
1065 	}
1066 	crp->crp_aad = cod->aad;
1067 	crp->crp_aad_start = 0;
1068 	crp->crp_aad_length = caead->aadlen;
1069 
1070 	if (cod->aad != NULL)
1071 		crp->crp_payload_start = 0;
1072 	else
1073 		crp->crp_payload_start = caead->aadlen;
1074 	error = copyin(caead->src, cod->buf + crp->crp_payload_start,
1075 	    caead->len);
1076 	if (error) {
1077 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1078 		goto bail;
1079 	}
1080 	crp->crp_payload_length = caead->len;
1081 	if (caead->op == COP_ENCRYPT && cod->obuf != NULL)
1082 		crp->crp_digest_start = crp->crp_payload_output_start +
1083 		    caead->len;
1084 	else
1085 		crp->crp_digest_start = crp->crp_payload_start + caead->len;
1086 
1087 	switch (cse->mode) {
1088 	case CSP_MODE_AEAD:
1089 	case CSP_MODE_ETA:
1090 		switch (caead->op) {
1091 		case COP_ENCRYPT:
1092 			crp->crp_op = CRYPTO_OP_ENCRYPT |
1093 			    CRYPTO_OP_COMPUTE_DIGEST;
1094 			break;
1095 		case COP_DECRYPT:
1096 			crp->crp_op = CRYPTO_OP_DECRYPT |
1097 			    CRYPTO_OP_VERIFY_DIGEST;
1098 			break;
1099 		default:
1100 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1101 			error = EINVAL;
1102 			goto bail;
1103 		}
1104 		break;
1105 	default:
1106 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1107 		error = EINVAL;
1108 		goto bail;
1109 	}
1110 
1111 	crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH);
1112 	crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len +
1113 	    cse->hashsize);
1114 	if (cod->obuf != NULL)
1115 		crypto_use_output_buf(crp, cod->obuf, caead->len +
1116 		    cse->hashsize);
1117 	crp->crp_callback = cryptodev_cb;
1118 	crp->crp_opaque = cod;
1119 
1120 	if (caead->iv) {
1121 		/*
1122 		 * Permit a 16-byte IV for AES-XTS, but only use the
1123 		 * first 8 bytes as a block number.
1124 		 */
1125 		if (cse->mode == CSP_MODE_ETA &&
1126 		    caead->ivlen == AES_BLOCK_LEN &&
1127 		    cse->ivsize == AES_XTS_IV_LEN)
1128 			caead->ivlen = AES_XTS_IV_LEN;
1129 
1130 		if (caead->ivlen != cse->ivsize) {
1131 			error = EINVAL;
1132 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1133 			goto bail;
1134 		}
1135 
1136 		error = copyin(caead->iv, crp->crp_iv, cse->ivsize);
1137 		if (error) {
1138 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1139 			goto bail;
1140 		}
1141 		crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
1142 	} else {
1143 		crp->crp_iv_start = crp->crp_payload_start;
1144 		crp->crp_payload_start += cse->ivsize;
1145 		crp->crp_payload_length -= cse->ivsize;
1146 		dst += cse->ivsize;
1147 	}
1148 
1149 	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
1150 		error = copyin(caead->tag, cod->buf + crp->crp_digest_start,
1151 		    cse->hashsize);
1152 		if (error) {
1153 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1154 			goto bail;
1155 		}
1156 	}
1157 again:
1158 	/*
1159 	 * Let the dispatch run unlocked, then, interlock against the
1160 	 * callback before checking if the operation completed and going
1161 	 * to sleep.  This insures drivers don't inherit our lock which
1162 	 * results in a lock order reversal between crypto_dispatch forced
1163 	 * entry and the crypto_done callback into us.
1164 	 */
1165 	error = crypto_dispatch(crp);
1166 	if (error != 0) {
1167 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1168 		goto bail;
1169 	}
1170 
1171 	mtx_lock(&cse->lock);
1172 	while (!cod->done)
1173 		mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
1174 	mtx_unlock(&cse->lock);
1175 
1176 	if (crp->crp_etype == EAGAIN) {
1177 		crp->crp_etype = 0;
1178 		crp->crp_flags &= ~CRYPTO_F_DONE;
1179 		cod->done = false;
1180 		goto again;
1181 	}
1182 
1183 	if (crp->crp_etype != 0) {
1184 		error = crp->crp_etype;
1185 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1186 		goto bail;
1187 	}
1188 
1189 	if (caead->dst != NULL) {
1190 		error = copyout(cod->obuf != NULL ? cod->obuf :
1191 		    cod->buf + crp->crp_payload_start, dst,
1192 		    crp->crp_payload_length);
1193 		if (error) {
1194 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1195 			goto bail;
1196 		}
1197 	}
1198 
1199 	if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) {
1200 		error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) +
1201 		    crp->crp_digest_start, caead->tag, cse->hashsize);
1202 		if (error) {
1203 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1204 			goto bail;
1205 		}
1206 	}
1207 
1208 bail:
1209 	crypto_freereq(crp);
1210 	cod_free(cod);
1211 
1212 	return (error);
1213 }
1214 
1215 static void
1216 cryptodevkey_cb(struct cryptkop *krp)
1217 {
1218 
1219 	wakeup_one(krp);
1220 }
1221 
1222 static int
1223 cryptodev_key(struct crypt_kop *kop)
1224 {
1225 	struct cryptkop *krp = NULL;
1226 	int error = EINVAL;
1227 	int in, out, size, i;
1228 
1229 	if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM) {
1230 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1231 		return (EFBIG);
1232 	}
1233 
1234 	in = kop->crk_iparams;
1235 	out = kop->crk_oparams;
1236 	switch (kop->crk_op) {
1237 	case CRK_MOD_EXP:
1238 		if (in == 3 && out == 1)
1239 			break;
1240 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1241 		return (EINVAL);
1242 	case CRK_MOD_EXP_CRT:
1243 		if (in == 6 && out == 1)
1244 			break;
1245 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1246 		return (EINVAL);
1247 	case CRK_DSA_SIGN:
1248 		if (in == 5 && out == 2)
1249 			break;
1250 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1251 		return (EINVAL);
1252 	case CRK_DSA_VERIFY:
1253 		if (in == 7 && out == 0)
1254 			break;
1255 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1256 		return (EINVAL);
1257 	case CRK_DH_COMPUTE_KEY:
1258 		if (in == 3 && out == 1)
1259 			break;
1260 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1261 		return (EINVAL);
1262 	default:
1263 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1264 		return (EINVAL);
1265 	}
1266 
1267 	krp = malloc(sizeof(*krp), M_XDATA, M_WAITOK | M_ZERO);
1268 	krp->krp_op = kop->crk_op;
1269 	krp->krp_status = kop->crk_status;
1270 	krp->krp_iparams = kop->crk_iparams;
1271 	krp->krp_oparams = kop->crk_oparams;
1272 	krp->krp_crid = kop->crk_crid;
1273 	krp->krp_status = 0;
1274 	krp->krp_callback = cryptodevkey_cb;
1275 
1276 	for (i = 0; i < CRK_MAXPARAM; i++) {
1277 		if (kop->crk_param[i].crp_nbits > 65536) {
1278 			/* Limit is the same as in OpenBSD */
1279 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1280 			goto fail;
1281 		}
1282 		krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits;
1283 	}
1284 	for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) {
1285 		size = (krp->krp_param[i].crp_nbits + 7) / 8;
1286 		if (size == 0)
1287 			continue;
1288 		krp->krp_param[i].crp_p = malloc(size, M_XDATA, M_WAITOK);
1289 		if (i >= krp->krp_iparams)
1290 			continue;
1291 		error = copyin(kop->crk_param[i].crp_p, krp->krp_param[i].crp_p, size);
1292 		if (error) {
1293 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1294 			goto fail;
1295 		}
1296 	}
1297 
1298 	error = crypto_kdispatch(krp);
1299 	if (error) {
1300 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1301 		goto fail;
1302 	}
1303 	error = tsleep(krp, PSOCK, "crydev", 0);
1304 	if (error) {
1305 		/* XXX can this happen?  if so, how do we recover? */
1306 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1307 		goto fail;
1308 	}
1309 
1310 	kop->crk_crid = krp->krp_hid;		/* device that did the work */
1311 	if (krp->krp_status != 0) {
1312 		error = krp->krp_status;
1313 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1314 		goto fail;
1315 	}
1316 
1317 	for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams; i++) {
1318 		size = (krp->krp_param[i].crp_nbits + 7) / 8;
1319 		if (size == 0)
1320 			continue;
1321 		error = copyout(krp->krp_param[i].crp_p, kop->crk_param[i].crp_p, size);
1322 		if (error) {
1323 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1324 			goto fail;
1325 		}
1326 	}
1327 
1328 fail:
1329 	if (krp) {
1330 		kop->crk_status = krp->krp_status;
1331 		for (i = 0; i < CRK_MAXPARAM; i++) {
1332 			if (krp->krp_param[i].crp_p)
1333 				free(krp->krp_param[i].crp_p, M_XDATA);
1334 		}
1335 		free(krp, M_XDATA);
1336 	}
1337 	return (error);
1338 }
1339 
1340 static int
1341 cryptodev_find(struct crypt_find_op *find)
1342 {
1343 	device_t dev;
1344 	size_t fnlen = sizeof find->name;
1345 
1346 	if (find->crid != -1) {
1347 		dev = crypto_find_device_byhid(find->crid);
1348 		if (dev == NULL)
1349 			return (ENOENT);
1350 		strncpy(find->name, device_get_nameunit(dev), fnlen);
1351 		find->name[fnlen - 1] = '\x0';
1352 	} else {
1353 		find->name[fnlen - 1] = '\x0';
1354 		find->crid = crypto_find_driver(find->name);
1355 		if (find->crid == -1)
1356 			return (ENOENT);
1357 	}
1358 	return (0);
1359 }
1360 
1361 static int
1362 cryptof_ioctl(struct file *fp, u_long cmd, void *data,
1363     struct ucred *active_cred, struct thread *td)
1364 {
1365 	static struct timeval keywarn, featwarn;
1366 	struct fcrypt *fcr = fp->f_data;
1367 	struct csession *cse;
1368 	struct session2_op *sop;
1369 	struct crypt_op *cop;
1370 	struct crypt_aead *caead;
1371 	struct crypt_kop *kop;
1372 	uint32_t ses;
1373 	int error = 0;
1374 	union {
1375 		struct session2_op sopc;
1376 #ifdef COMPAT_FREEBSD32
1377 		struct crypt_op copc;
1378 		struct crypt_aead aeadc;
1379 		struct crypt_kop kopc;
1380 #endif
1381 	} thunk;
1382 #ifdef COMPAT_FREEBSD32
1383 	u_long cmd32;
1384 	void *data32;
1385 
1386 	cmd32 = 0;
1387 	data32 = NULL;
1388 	switch (cmd) {
1389 	case CIOCGSESSION32:
1390 		cmd32 = cmd;
1391 		data32 = data;
1392 		cmd = CIOCGSESSION;
1393 		data = &thunk.sopc;
1394 		session_op_from_32((struct session_op32 *)data32, &thunk.sopc);
1395 		break;
1396 	case CIOCGSESSION232:
1397 		cmd32 = cmd;
1398 		data32 = data;
1399 		cmd = CIOCGSESSION2;
1400 		data = &thunk.sopc;
1401 		session2_op_from_32((struct session2_op32 *)data32,
1402 		    &thunk.sopc);
1403 		break;
1404 	case CIOCCRYPT32:
1405 		cmd32 = cmd;
1406 		data32 = data;
1407 		cmd = CIOCCRYPT;
1408 		data = &thunk.copc;
1409 		crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc);
1410 		break;
1411 	case CIOCCRYPTAEAD32:
1412 		cmd32 = cmd;
1413 		data32 = data;
1414 		cmd = CIOCCRYPTAEAD;
1415 		data = &thunk.aeadc;
1416 		crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc);
1417 		break;
1418 	case CIOCKEY32:
1419 	case CIOCKEY232:
1420 		cmd32 = cmd;
1421 		data32 = data;
1422 		if (cmd == CIOCKEY32)
1423 			cmd = CIOCKEY;
1424 		else
1425 			cmd = CIOCKEY2;
1426 		data = &thunk.kopc;
1427 		crypt_kop_from_32((struct crypt_kop32 *)data32, &thunk.kopc);
1428 		break;
1429 	}
1430 #endif
1431 
1432 	switch (cmd) {
1433 	case CIOCGSESSION:
1434 	case CIOCGSESSION2:
1435 		if (cmd == CIOCGSESSION) {
1436 			session2_op_from_op(data, &thunk.sopc);
1437 			sop = &thunk.sopc;
1438 		} else
1439 			sop = (struct session2_op *)data;
1440 
1441 		error = cse_create(fcr, sop);
1442 		if (cmd == CIOCGSESSION && error == 0)
1443 			session2_op_to_op(sop, data);
1444 		break;
1445 	case CIOCFSESSION:
1446 		ses = *(uint32_t *)data;
1447 		if (!cse_delete(fcr, ses)) {
1448 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1449 			return (EINVAL);
1450 		}
1451 		break;
1452 	case CIOCCRYPT:
1453 		cop = (struct crypt_op *)data;
1454 		cse = cse_find(fcr, cop->ses);
1455 		if (cse == NULL) {
1456 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1457 			return (EINVAL);
1458 		}
1459 		error = cryptodev_op(cse, cop, active_cred, td);
1460 		cse_free(cse);
1461 		break;
1462 	case CIOCKEY:
1463 	case CIOCKEY2:
1464 		if (ratecheck(&keywarn, &warninterval))
1465 			gone_in(14,
1466 			    "Asymmetric crypto operations via /dev/crypto");
1467 
1468 		if (!crypto_userasymcrypto) {
1469 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1470 			return (EPERM);		/* XXX compat? */
1471 		}
1472 		kop = (struct crypt_kop *)data;
1473 		if (cmd == CIOCKEY) {
1474 			/* NB: crypto core enforces s/w driver use */
1475 			kop->crk_crid =
1476 			    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE;
1477 		}
1478 		mtx_lock(&Giant);
1479 		error = cryptodev_key(kop);
1480 		mtx_unlock(&Giant);
1481 		break;
1482 	case CIOCASYMFEAT:
1483 		if (ratecheck(&featwarn, &warninterval))
1484 			gone_in(14,
1485 			    "Asymmetric crypto features via /dev/crypto");
1486 
1487 		if (!crypto_userasymcrypto) {
1488 			/*
1489 			 * NB: if user asym crypto operations are
1490 			 * not permitted return "no algorithms"
1491 			 * so well-behaved applications will just
1492 			 * fallback to doing them in software.
1493 			 */
1494 			*(int *)data = 0;
1495 		} else {
1496 			error = crypto_getfeat((int *)data);
1497 			if (error)
1498 				SDT_PROBE1(opencrypto, dev, ioctl, error,
1499 				    __LINE__);
1500 		}
1501 		break;
1502 	case CIOCFINDDEV:
1503 		error = cryptodev_find((struct crypt_find_op *)data);
1504 		break;
1505 	case CIOCCRYPTAEAD:
1506 		caead = (struct crypt_aead *)data;
1507 		cse = cse_find(fcr, caead->ses);
1508 		if (cse == NULL) {
1509 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1510 			return (EINVAL);
1511 		}
1512 		error = cryptodev_aead(cse, caead, active_cred, td);
1513 		cse_free(cse);
1514 		break;
1515 	default:
1516 		error = EINVAL;
1517 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1518 		break;
1519 	}
1520 
1521 #ifdef COMPAT_FREEBSD32
1522 	switch (cmd32) {
1523 	case CIOCGSESSION32:
1524 		if (error == 0)
1525 			session_op_to_32(data, data32);
1526 		break;
1527 	case CIOCGSESSION232:
1528 		if (error == 0)
1529 			session2_op_to_32(data, data32);
1530 		break;
1531 	case CIOCCRYPT32:
1532 		if (error == 0)
1533 			crypt_op_to_32(data, data32);
1534 		break;
1535 	case CIOCCRYPTAEAD32:
1536 		if (error == 0)
1537 			crypt_aead_to_32(data, data32);
1538 		break;
1539 	case CIOCKEY32:
1540 	case CIOCKEY232:
1541 		crypt_kop_to_32(data, data32);
1542 		break;
1543 	}
1544 #endif
1545 	return (error);
1546 }
1547 
1548 /* ARGSUSED */
1549 static int
1550 cryptof_stat(struct file *fp, struct stat *sb, struct ucred *active_cred,
1551     struct thread *td)
1552 {
1553 
1554 	return (EOPNOTSUPP);
1555 }
1556 
1557 /* ARGSUSED */
1558 static int
1559 cryptof_close(struct file *fp, struct thread *td)
1560 {
1561 	struct fcrypt *fcr = fp->f_data;
1562 	struct csession *cse;
1563 
1564 	while ((cse = TAILQ_FIRST(&fcr->csessions))) {
1565 		TAILQ_REMOVE(&fcr->csessions, cse, next);
1566 		KASSERT(cse->refs == 1,
1567 		    ("%s: crypto session %p with %d refs", __func__, cse,
1568 		    cse->refs));
1569 		cse_free(cse);
1570 	}
1571 	free(fcr, M_XDATA);
1572 	fp->f_data = NULL;
1573 	return 0;
1574 }
1575 
1576 static int
1577 cryptof_fill_kinfo(struct file *fp, struct kinfo_file *kif,
1578     struct filedesc *fdp)
1579 {
1580 
1581 	kif->kf_type = KF_TYPE_CRYPTO;
1582 	return (0);
1583 }
1584 
1585 static int
1586 cryptoioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag,
1587     struct thread *td)
1588 {
1589 	struct file *f;
1590 	struct fcrypt *fcr;
1591 	int fd, error;
1592 
1593 	switch (cmd) {
1594 	case CRIOGET:
1595 		error = falloc_noinstall(td, &f);
1596 		if (error)
1597 			break;
1598 
1599 		fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO);
1600 		TAILQ_INIT(&fcr->csessions);
1601 		mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF);
1602 
1603 		finit(f, FREAD | FWRITE, DTYPE_CRYPTO, fcr, &cryptofops);
1604 		error = finstall(td, f, &fd, 0, NULL);
1605 		if (error) {
1606 			mtx_destroy(&fcr->lock);
1607 			free(fcr, M_XDATA);
1608 		} else
1609 			*(uint32_t *)data = fd;
1610 		fdrop(f, td);
1611 		break;
1612 	case CRIOFINDDEV:
1613 		error = cryptodev_find((struct crypt_find_op *)data);
1614 		break;
1615 	case CRIOASYMFEAT:
1616 		error = crypto_getfeat((int *)data);
1617 		break;
1618 	default:
1619 		error = EINVAL;
1620 		break;
1621 	}
1622 	return (error);
1623 }
1624 
1625 static struct cdevsw crypto_cdevsw = {
1626 	.d_version =	D_VERSION,
1627 	.d_ioctl =	cryptoioctl,
1628 	.d_name =	"crypto",
1629 };
1630 static struct cdev *crypto_dev;
1631 
1632 /*
1633  * Initialization code, both for static and dynamic loading.
1634  */
1635 static int
1636 cryptodev_modevent(module_t mod, int type, void *unused)
1637 {
1638 	switch (type) {
1639 	case MOD_LOAD:
1640 		if (bootverbose)
1641 			printf("crypto: <crypto device>\n");
1642 		crypto_dev = make_dev(&crypto_cdevsw, 0,
1643 				      UID_ROOT, GID_WHEEL, 0666,
1644 				      "crypto");
1645 		return 0;
1646 	case MOD_UNLOAD:
1647 		/*XXX disallow if active sessions */
1648 		destroy_dev(crypto_dev);
1649 		return 0;
1650 	}
1651 	return EINVAL;
1652 }
1653 
1654 static moduledata_t cryptodev_mod = {
1655 	"cryptodev",
1656 	cryptodev_modevent,
1657 	0
1658 };
1659 MODULE_VERSION(cryptodev, 1);
1660 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
1661 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1);
1662 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1);
1663