xref: /freebsd/sys/opencrypto/crypto.c (revision 70439285ad9d5ec8ebdb1cf8617e8f30ea31b2e2)
16810ad6fSSam Leffler /*-
26810ad6fSSam Leffler  * Copyright (c) 2002-2006 Sam Leffler.  All rights reserved.
3ae18720dSJohn Baldwin  * Copyright (c) 2021 The FreeBSD Foundation
4ae18720dSJohn Baldwin  *
5ae18720dSJohn Baldwin  * Portions of this software were developed by Ararat River
6ae18720dSJohn Baldwin  * Consulting, LLC under sponsorship of the FreeBSD Foundation.
76810ad6fSSam Leffler  *
86810ad6fSSam Leffler  * Redistribution and use in source and binary forms, with or without
96810ad6fSSam Leffler  * modification, are permitted provided that the following conditions
106810ad6fSSam Leffler  * are met:
116810ad6fSSam Leffler  * 1. Redistributions of source code must retain the above copyright
126810ad6fSSam Leffler  *    notice, this list of conditions and the following disclaimer.
136810ad6fSSam Leffler  * 2. Redistributions in binary form must reproduce the above copyright
146810ad6fSSam Leffler  *    notice, this list of conditions and the following disclaimer in the
156810ad6fSSam Leffler  *    documentation and/or other materials provided with the distribution.
166810ad6fSSam Leffler  *
176810ad6fSSam Leffler  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
186810ad6fSSam Leffler  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
196810ad6fSSam Leffler  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
206810ad6fSSam Leffler  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
216810ad6fSSam Leffler  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
226810ad6fSSam Leffler  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
236810ad6fSSam Leffler  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
246810ad6fSSam Leffler  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
256810ad6fSSam Leffler  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
266810ad6fSSam Leffler  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
276810ad6fSSam Leffler  */
286810ad6fSSam Leffler 
296810ad6fSSam Leffler #include <sys/cdefs.h>
306810ad6fSSam Leffler __FBSDID("$FreeBSD$");
316810ad6fSSam Leffler 
326810ad6fSSam Leffler /*
336810ad6fSSam Leffler  * Cryptographic Subsystem.
346810ad6fSSam Leffler  *
356810ad6fSSam Leffler  * This code is derived from the Openbsd Cryptographic Framework (OCF)
366810ad6fSSam Leffler  * that has the copyright shown below.  Very little of the original
376810ad6fSSam Leffler  * code remains.
386810ad6fSSam Leffler  */
396810ad6fSSam Leffler 
4060727d8bSWarner Losh /*-
41091d81d1SSam Leffler  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
42091d81d1SSam Leffler  *
43091d81d1SSam Leffler  * This code was written by Angelos D. Keromytis in Athens, Greece, in
44091d81d1SSam Leffler  * February 2000. Network Security Technologies Inc. (NSTI) kindly
45091d81d1SSam Leffler  * supported the development of this code.
46091d81d1SSam Leffler  *
47091d81d1SSam Leffler  * Copyright (c) 2000, 2001 Angelos D. Keromytis
48091d81d1SSam Leffler  *
49091d81d1SSam Leffler  * Permission to use, copy, and modify this software with or without fee
50091d81d1SSam Leffler  * is hereby granted, provided that this entire notice is included in
51091d81d1SSam Leffler  * all source code copies of any software which is or includes a copy or
52091d81d1SSam Leffler  * modification of this software.
53091d81d1SSam Leffler  *
54091d81d1SSam Leffler  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
55091d81d1SSam Leffler  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
56091d81d1SSam Leffler  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
57091d81d1SSam Leffler  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
58091d81d1SSam Leffler  * PURPOSE.
59091d81d1SSam Leffler  */
602c446514SDavid E. O'Brien 
61c0341432SJohn Baldwin #include "opt_compat.h"
626810ad6fSSam Leffler #include "opt_ddb.h"
636810ad6fSSam Leffler 
64091d81d1SSam Leffler #include <sys/param.h>
65091d81d1SSam Leffler #include <sys/systm.h>
667290cb47SMark Johnston #include <sys/counter.h>
67091d81d1SSam Leffler #include <sys/kernel.h>
68091d81d1SSam Leffler #include <sys/kthread.h>
69ec5c0e5bSAllan Jude #include <sys/linker.h>
70091d81d1SSam Leffler #include <sys/lock.h>
715dba30f1SPoul-Henning Kamp #include <sys/module.h>
72091d81d1SSam Leffler #include <sys/mutex.h>
73091d81d1SSam Leffler #include <sys/malloc.h>
749c0e3d3aSJohn Baldwin #include <sys/mbuf.h>
75091d81d1SSam Leffler #include <sys/proc.h>
76c0341432SJohn Baldwin #include <sys/refcount.h>
77df21ad6eSBjoern A. Zeeb #include <sys/sdt.h>
7839bbca6fSFabien Thomas #include <sys/smp.h>
79091d81d1SSam Leffler #include <sys/sysctl.h>
8039bbca6fSFabien Thomas #include <sys/taskqueue.h>
819c0e3d3aSJohn Baldwin #include <sys/uio.h>
82091d81d1SSam Leffler 
836810ad6fSSam Leffler #include <ddb/ddb.h>
846810ad6fSSam Leffler 
85e6f6d0c9SAlan Somers #include <machine/vmparam.h>
86091d81d1SSam Leffler #include <vm/uma.h>
87e6f6d0c9SAlan Somers 
88ec5c0e5bSAllan Jude #include <crypto/intake.h>
89091d81d1SSam Leffler #include <opencrypto/cryptodev.h>
90c0341432SJohn Baldwin #include <opencrypto/xform_auth.h>
91c0341432SJohn Baldwin #include <opencrypto/xform_enc.h>
92091d81d1SSam Leffler 
936810ad6fSSam Leffler #include <sys/kobj.h>
946810ad6fSSam Leffler #include <sys/bus.h>
956810ad6fSSam Leffler #include "cryptodev_if.h"
966810ad6fSSam Leffler 
976ed982a2SAndrew Turner #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
9804c49e68SKonstantin Belousov #include <machine/pcb.h>
9904c49e68SKonstantin Belousov #endif
10004c49e68SKonstantin Belousov 
101df21ad6eSBjoern A. Zeeb SDT_PROVIDER_DEFINE(opencrypto);
102df21ad6eSBjoern A. Zeeb 
103091d81d1SSam Leffler /*
104091d81d1SSam Leffler  * Crypto drivers register themselves by allocating a slot in the
10576681661SJohn Baldwin  * crypto_drivers table with crypto_get_driverid().
106091d81d1SSam Leffler  */
107091d81d1SSam Leffler static	struct mtx crypto_drivers_mtx;		/* lock on driver table */
108091d81d1SSam Leffler #define	CRYPTO_DRIVER_LOCK()	mtx_lock(&crypto_drivers_mtx)
109091d81d1SSam Leffler #define	CRYPTO_DRIVER_UNLOCK()	mtx_unlock(&crypto_drivers_mtx)
1106810ad6fSSam Leffler #define	CRYPTO_DRIVER_ASSERT()	mtx_assert(&crypto_drivers_mtx, MA_OWNED)
1116810ad6fSSam Leffler 
1126810ad6fSSam Leffler /*
1136810ad6fSSam Leffler  * Crypto device/driver capabilities structure.
1146810ad6fSSam Leffler  *
1156810ad6fSSam Leffler  * Synchronization:
1166810ad6fSSam Leffler  * (d) - protected by CRYPTO_DRIVER_LOCK()
1176810ad6fSSam Leffler  * (q) - protected by CRYPTO_Q_LOCK()
1186810ad6fSSam Leffler  * Not tagged fields are read-only.
1196810ad6fSSam Leffler  */
1206810ad6fSSam Leffler struct cryptocap {
121c0341432SJohn Baldwin 	device_t	cc_dev;
122c0341432SJohn Baldwin 	uint32_t	cc_hid;
123d3d79e96SJohn Baldwin 	uint32_t	cc_sessions;		/* (d) # of sessions */
1246810ad6fSSam Leffler 
1256810ad6fSSam Leffler 	int		cc_flags;		/* (d) flags */
1266810ad6fSSam Leffler #define CRYPTOCAP_F_CLEANUP	0x80000000	/* needs resource cleanup */
1276810ad6fSSam Leffler 	int		cc_qblocked;		/* (q) symmetric q blocked */
1281b0909d5SConrad Meyer 	size_t		cc_session_size;
129c0341432SJohn Baldwin 	volatile int	cc_refs;
1306810ad6fSSam Leffler };
131c0341432SJohn Baldwin 
132c0341432SJohn Baldwin static	struct cryptocap **crypto_drivers = NULL;
133c0341432SJohn Baldwin static	int crypto_drivers_size = 0;
134c0341432SJohn Baldwin 
135c0341432SJohn Baldwin struct crypto_session {
136c0341432SJohn Baldwin 	struct cryptocap *cap;
137c0341432SJohn Baldwin 	struct crypto_session_params csp;
13898d788c8SMark Johnston 	uint64_t id;
1398adcc757SMark Johnston 	/* Driver softc follows. */
140c0341432SJohn Baldwin };
141091d81d1SSam Leffler 
1423a865c82SPawel Jakub Dawidek static	int crp_sleep = 0;
14339bbca6fSFabien Thomas static	TAILQ_HEAD(cryptop_q ,cryptop) crp_q;		/* request queues */
144091d81d1SSam Leffler static	struct mtx crypto_q_mtx;
145091d81d1SSam Leffler #define	CRYPTO_Q_LOCK()		mtx_lock(&crypto_q_mtx)
146091d81d1SSam Leffler #define	CRYPTO_Q_UNLOCK()	mtx_unlock(&crypto_q_mtx)
147091d81d1SSam Leffler 
14833f3bad3SJohn Baldwin SYSCTL_NODE(_kern, OID_AUTO, crypto, CTLFLAG_RW, 0,
149c0341432SJohn Baldwin     "In-kernel cryptography");
150c0341432SJohn Baldwin 
151091d81d1SSam Leffler /*
15239bbca6fSFabien Thomas  * Taskqueue used to dispatch the crypto requests
15339bbca6fSFabien Thomas  * that have the CRYPTO_F_ASYNC flag
154091d81d1SSam Leffler  */
15539bbca6fSFabien Thomas static struct taskqueue *crypto_tq;
15639bbca6fSFabien Thomas 
15739bbca6fSFabien Thomas /*
15839bbca6fSFabien Thomas  * Crypto seq numbers are operated on with modular arithmetic
15939bbca6fSFabien Thomas  */
16039bbca6fSFabien Thomas #define	CRYPTO_SEQ_GT(a,b)	((int)((a)-(b)) > 0)
16139bbca6fSFabien Thomas 
16239bbca6fSFabien Thomas struct crypto_ret_worker {
16339bbca6fSFabien Thomas 	struct mtx crypto_ret_mtx;
16439bbca6fSFabien Thomas 
16539bbca6fSFabien Thomas 	TAILQ_HEAD(,cryptop) crp_ordered_ret_q;	/* ordered callback queue for symetric jobs */
16639bbca6fSFabien Thomas 	TAILQ_HEAD(,cryptop) crp_ret_q;		/* callback queue for symetric jobs */
16739bbca6fSFabien Thomas 
168d3d79e96SJohn Baldwin 	uint32_t reorder_ops;		/* total ordered sym jobs received */
169d3d79e96SJohn Baldwin 	uint32_t reorder_cur_seq;	/* current sym job dispatched */
17039bbca6fSFabien Thomas 
17171785781SJohn Baldwin 	struct thread *td;
17239bbca6fSFabien Thomas };
17339bbca6fSFabien Thomas static struct crypto_ret_worker *crypto_ret_workers = NULL;
17439bbca6fSFabien Thomas 
17539bbca6fSFabien Thomas #define CRYPTO_RETW(i)		(&crypto_ret_workers[i])
17639bbca6fSFabien Thomas #define CRYPTO_RETW_ID(w)	((w) - crypto_ret_workers)
17739bbca6fSFabien Thomas #define FOREACH_CRYPTO_RETW(w) \
17839bbca6fSFabien Thomas 	for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w)
17939bbca6fSFabien Thomas 
18039bbca6fSFabien Thomas #define	CRYPTO_RETW_LOCK(w)	mtx_lock(&w->crypto_ret_mtx)
18139bbca6fSFabien Thomas #define	CRYPTO_RETW_UNLOCK(w)	mtx_unlock(&w->crypto_ret_mtx)
18239bbca6fSFabien Thomas 
18339bbca6fSFabien Thomas static int crypto_workers_num = 0;
184c0341432SJohn Baldwin SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN,
185c0341432SJohn Baldwin 	   &crypto_workers_num, 0,
186c0341432SJohn Baldwin 	   "Number of crypto workers used to dispatch crypto jobs");
187c0341432SJohn Baldwin #ifdef COMPAT_FREEBSD12
18839bbca6fSFabien Thomas SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN,
18939bbca6fSFabien Thomas 	   &crypto_workers_num, 0,
19039bbca6fSFabien Thomas 	   "Number of crypto workers used to dispatch crypto jobs");
191c0341432SJohn Baldwin #endif
192091d81d1SSam Leffler 
193091d81d1SSam Leffler static	uma_zone_t cryptop_zone;
194091d81d1SSam Leffler 
195c0341432SJohn Baldwin int	crypto_devallowsoft = 0;
1969e0c0512SMark Johnston SYSCTL_INT(_kern_crypto, OID_AUTO, allow_soft, CTLFLAG_RWTUN,
197c0341432SJohn Baldwin 	   &crypto_devallowsoft, 0,
198c0341432SJohn Baldwin 	   "Enable use of software crypto by /dev/crypto");
199c0341432SJohn Baldwin #ifdef COMPAT_FREEBSD12
2009e0c0512SMark Johnston SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RWTUN,
201091d81d1SSam Leffler 	   &crypto_devallowsoft, 0,
2026c20d7a3SJohn-Mark Gurney 	   "Enable/disable use of software crypto by /dev/crypto");
203c0341432SJohn Baldwin #endif
204091d81d1SSam Leffler 
205*70439285SMateusz Guzik #ifdef DIAGNOSTIC
206*70439285SMateusz Guzik bool crypto_destroyreq_check;
207*70439285SMateusz Guzik SYSCTL_BOOL(_kern_crypto, OID_AUTO, destroyreq_check, CTLFLAG_RWTUN,
208*70439285SMateusz Guzik 	   &crypto_destroyreq_check, 0,
209*70439285SMateusz Guzik 	   "Enable checks when destroying a request");
210*70439285SMateusz Guzik #endif
211*70439285SMateusz Guzik 
212091d81d1SSam Leffler MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
213091d81d1SSam Leffler 
21471785781SJohn Baldwin static	void crypto_dispatch_thread(void *arg);
21571785781SJohn Baldwin static	struct thread *cryptotd;
21671785781SJohn Baldwin static	void crypto_ret_thread(void *arg);
21751e45326SSam Leffler static	void crypto_destroy(void);
2184acae0acSPawel Jakub Dawidek static	int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
21939bbca6fSFabien Thomas static	void crypto_task_invoke(void *ctx, int pending);
22039bbca6fSFabien Thomas static void crypto_batch_enqueue(struct cryptop *crp);
22151e45326SSam Leffler 
2227290cb47SMark Johnston static counter_u64_t cryptostats[sizeof(struct cryptostats) / sizeof(uint64_t)];
2237290cb47SMark Johnston SYSCTL_COUNTER_U64_ARRAY(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW,
2247290cb47SMark Johnston     cryptostats, nitems(cryptostats),
2257290cb47SMark Johnston     "Crypto system statistics");
2267290cb47SMark Johnston 
2277290cb47SMark Johnston #define	CRYPTOSTAT_INC(stat) do {					\
2287290cb47SMark Johnston 	counter_u64_add(						\
2297290cb47SMark Johnston 	    cryptostats[offsetof(struct cryptostats, stat) / sizeof(uint64_t)],\
2307290cb47SMark Johnston 	    1);								\
2317290cb47SMark Johnston } while (0)
2327290cb47SMark Johnston 
2337290cb47SMark Johnston static void
2347290cb47SMark Johnston cryptostats_init(void *arg __unused)
2357290cb47SMark Johnston {
2367290cb47SMark Johnston 	COUNTER_ARRAY_ALLOC(cryptostats, nitems(cryptostats), M_WAITOK);
2377290cb47SMark Johnston }
2387290cb47SMark Johnston SYSINIT(cryptostats_init, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_init, NULL);
2397290cb47SMark Johnston 
2407290cb47SMark Johnston static void
2417290cb47SMark Johnston cryptostats_fini(void *arg __unused)
2427290cb47SMark Johnston {
2437290cb47SMark Johnston 	COUNTER_ARRAY_FREE(cryptostats, nitems(cryptostats));
2447290cb47SMark Johnston }
2457290cb47SMark Johnston SYSUNINIT(cryptostats_fini, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_fini,
2467290cb47SMark Johnston     NULL);
2477d1853eeSSam Leffler 
248ec5c0e5bSAllan Jude /* Try to avoid directly exposing the key buffer as a symbol */
249ec5c0e5bSAllan Jude static struct keybuf *keybuf;
250ec5c0e5bSAllan Jude 
251ec5c0e5bSAllan Jude static struct keybuf empty_keybuf = {
252ec5c0e5bSAllan Jude         .kb_nents = 0
253ec5c0e5bSAllan Jude };
254ec5c0e5bSAllan Jude 
255ec5c0e5bSAllan Jude /* Obtain the key buffer from boot metadata */
256ec5c0e5bSAllan Jude static void
257ec5c0e5bSAllan Jude keybuf_init(void)
258ec5c0e5bSAllan Jude {
259ec5c0e5bSAllan Jude 	caddr_t kmdp;
260ec5c0e5bSAllan Jude 
261ec5c0e5bSAllan Jude 	kmdp = preload_search_by_type("elf kernel");
262ec5c0e5bSAllan Jude 
263ec5c0e5bSAllan Jude 	if (kmdp == NULL)
264ec5c0e5bSAllan Jude 		kmdp = preload_search_by_type("elf64 kernel");
265ec5c0e5bSAllan Jude 
266ec5c0e5bSAllan Jude 	keybuf = (struct keybuf *)preload_search_info(kmdp,
267ec5c0e5bSAllan Jude 	    MODINFO_METADATA | MODINFOMD_KEYBUF);
268ec5c0e5bSAllan Jude 
269ec5c0e5bSAllan Jude         if (keybuf == NULL)
270ec5c0e5bSAllan Jude                 keybuf = &empty_keybuf;
271ec5c0e5bSAllan Jude }
272ec5c0e5bSAllan Jude 
273ec5c0e5bSAllan Jude /* It'd be nice if we could store these in some kind of secure memory... */
2745973f492SJohn Baldwin struct keybuf *
2755973f492SJohn Baldwin get_keybuf(void)
2765973f492SJohn Baldwin {
277ec5c0e5bSAllan Jude 
278ec5c0e5bSAllan Jude         return (keybuf);
279ec5c0e5bSAllan Jude }
280ec5c0e5bSAllan Jude 
281c0341432SJohn Baldwin static struct cryptocap *
282c0341432SJohn Baldwin cap_ref(struct cryptocap *cap)
283c0341432SJohn Baldwin {
284c0341432SJohn Baldwin 
285c0341432SJohn Baldwin 	refcount_acquire(&cap->cc_refs);
286c0341432SJohn Baldwin 	return (cap);
287c0341432SJohn Baldwin }
288c0341432SJohn Baldwin 
289c0341432SJohn Baldwin static void
290c0341432SJohn Baldwin cap_rele(struct cryptocap *cap)
291c0341432SJohn Baldwin {
292c0341432SJohn Baldwin 
293c0341432SJohn Baldwin 	if (refcount_release(&cap->cc_refs) == 0)
294c0341432SJohn Baldwin 		return;
295c0341432SJohn Baldwin 
296c0341432SJohn Baldwin 	KASSERT(cap->cc_sessions == 0,
297c0341432SJohn Baldwin 	    ("freeing crypto driver with active sessions"));
298c0341432SJohn Baldwin 
299c0341432SJohn Baldwin 	free(cap, M_CRYPTO_DATA);
300c0341432SJohn Baldwin }
301c0341432SJohn Baldwin 
30251e45326SSam Leffler static int
303091d81d1SSam Leffler crypto_init(void)
304091d81d1SSam Leffler {
30539bbca6fSFabien Thomas 	struct crypto_ret_worker *ret_worker;
30671785781SJohn Baldwin 	struct proc *p;
30751e45326SSam Leffler 	int error;
308091d81d1SSam Leffler 
3094e057806SJohn Baldwin 	mtx_init(&crypto_drivers_mtx, "crypto driver table", NULL, MTX_DEF);
310091d81d1SSam Leffler 
311091d81d1SSam Leffler 	TAILQ_INIT(&crp_q);
3124e057806SJohn Baldwin 	mtx_init(&crypto_q_mtx, "crypto op queues", NULL, MTX_DEF);
313091d81d1SSam Leffler 
314e5587cbbSMark Johnston 	cryptop_zone = uma_zcreate("cryptop",
315e5587cbbSMark Johnston 	    sizeof(struct cryptop), NULL, NULL, NULL, NULL,
31651e45326SSam Leffler 	    UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
3171b0909d5SConrad Meyer 
318c0341432SJohn Baldwin 	crypto_drivers_size = CRYPTO_DRIVERS_INITIAL;
319c0341432SJohn Baldwin 	crypto_drivers = malloc(crypto_drivers_size *
320e5587cbbSMark Johnston 	    sizeof(struct cryptocap), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
32151e45326SSam Leffler 
32239bbca6fSFabien Thomas 	if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus)
32339bbca6fSFabien Thomas 		crypto_workers_num = mp_ncpus;
32439bbca6fSFabien Thomas 
32539bbca6fSFabien Thomas 	crypto_tq = taskqueue_create("crypto", M_WAITOK | M_ZERO,
32639bbca6fSFabien Thomas 	    taskqueue_thread_enqueue, &crypto_tq);
32739bbca6fSFabien Thomas 
32839bbca6fSFabien Thomas 	taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN,
32939bbca6fSFabien Thomas 	    "crypto");
33039bbca6fSFabien Thomas 
33171785781SJohn Baldwin 	p = NULL;
33271785781SJohn Baldwin 	error = kproc_kthread_add(crypto_dispatch_thread, NULL, &p, &cryptotd,
33371785781SJohn Baldwin 	    0, 0, "crypto", "crypto");
33451e45326SSam Leffler 	if (error) {
33551e45326SSam Leffler 		printf("crypto_init: cannot start crypto thread; error %d",
33651e45326SSam Leffler 			error);
33751e45326SSam Leffler 		goto bad;
33851e45326SSam Leffler 	}
33951e45326SSam Leffler 
340e5587cbbSMark Johnston 	crypto_ret_workers = mallocarray(crypto_workers_num,
341e5587cbbSMark Johnston 	    sizeof(struct crypto_ret_worker), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
34239bbca6fSFabien Thomas 
34339bbca6fSFabien Thomas 	FOREACH_CRYPTO_RETW(ret_worker) {
34439bbca6fSFabien Thomas 		TAILQ_INIT(&ret_worker->crp_ordered_ret_q);
34539bbca6fSFabien Thomas 		TAILQ_INIT(&ret_worker->crp_ret_q);
34639bbca6fSFabien Thomas 
34739bbca6fSFabien Thomas 		ret_worker->reorder_ops = 0;
34839bbca6fSFabien Thomas 		ret_worker->reorder_cur_seq = 0;
34939bbca6fSFabien Thomas 
3504e057806SJohn Baldwin 		mtx_init(&ret_worker->crypto_ret_mtx, "crypto return queues",
3514e057806SJohn Baldwin 		    NULL, MTX_DEF);
35239bbca6fSFabien Thomas 
35371785781SJohn Baldwin 		error = kthread_add(crypto_ret_thread, ret_worker, p,
35471785781SJohn Baldwin 		    &ret_worker->td, 0, 0, "crypto returns %td",
35571785781SJohn Baldwin 		    CRYPTO_RETW_ID(ret_worker));
35651e45326SSam Leffler 		if (error) {
35751e45326SSam Leffler 			printf("crypto_init: cannot start cryptoret thread; error %d",
35851e45326SSam Leffler 				error);
35951e45326SSam Leffler 			goto bad;
36051e45326SSam Leffler 		}
36139bbca6fSFabien Thomas 	}
362ec5c0e5bSAllan Jude 
363ec5c0e5bSAllan Jude 	keybuf_init();
364ec5c0e5bSAllan Jude 
36551e45326SSam Leffler 	return 0;
36651e45326SSam Leffler bad:
36751e45326SSam Leffler 	crypto_destroy();
36851e45326SSam Leffler 	return error;
36951e45326SSam Leffler }
37051e45326SSam Leffler 
37151e45326SSam Leffler /*
37251e45326SSam Leffler  * Signal a crypto thread to terminate.  We use the driver
37351e45326SSam Leffler  * table lock to synchronize the sleep/wakeups so that we
37451e45326SSam Leffler  * are sure the threads have terminated before we release
37551e45326SSam Leffler  * the data structures they use.  See crypto_finis below
37651e45326SSam Leffler  * for the other half of this song-and-dance.
37751e45326SSam Leffler  */
37851e45326SSam Leffler static void
37971785781SJohn Baldwin crypto_terminate(struct thread **tdp, void *q)
38051e45326SSam Leffler {
38171785781SJohn Baldwin 	struct thread *td;
38251e45326SSam Leffler 
38351e45326SSam Leffler 	mtx_assert(&crypto_drivers_mtx, MA_OWNED);
38471785781SJohn Baldwin 	td = *tdp;
38571785781SJohn Baldwin 	*tdp = NULL;
38671785781SJohn Baldwin 	if (td != NULL) {
38751e45326SSam Leffler 		wakeup_one(q);
38871785781SJohn Baldwin 		mtx_sleep(td, &crypto_drivers_mtx, PWAIT, "crypto_destroy", 0);
38951e45326SSam Leffler 	}
39051e45326SSam Leffler }
39151e45326SSam Leffler 
39251e45326SSam Leffler static void
393d588dc7dSMark Johnston hmac_init_pad(const struct auth_hash *axf, const char *key, int klen,
394d588dc7dSMark Johnston     void *auth_ctx, uint8_t padval)
395c0341432SJohn Baldwin {
396c0341432SJohn Baldwin 	uint8_t hmac_key[HMAC_MAX_BLOCK_LEN];
397c0341432SJohn Baldwin 	u_int i;
398c0341432SJohn Baldwin 
399c0341432SJohn Baldwin 	KASSERT(axf->blocksize <= sizeof(hmac_key),
400c0341432SJohn Baldwin 	    ("Invalid HMAC block size %d", axf->blocksize));
401c0341432SJohn Baldwin 
402c0341432SJohn Baldwin 	/*
403c0341432SJohn Baldwin 	 * If the key is larger than the block size, use the digest of
404c0341432SJohn Baldwin 	 * the key as the key instead.
405c0341432SJohn Baldwin 	 */
406c0341432SJohn Baldwin 	memset(hmac_key, 0, sizeof(hmac_key));
407c0341432SJohn Baldwin 	if (klen > axf->blocksize) {
408c0341432SJohn Baldwin 		axf->Init(auth_ctx);
409c0341432SJohn Baldwin 		axf->Update(auth_ctx, key, klen);
410c0341432SJohn Baldwin 		axf->Final(hmac_key, auth_ctx);
411c0341432SJohn Baldwin 		klen = axf->hashsize;
412c0341432SJohn Baldwin 	} else
413c0341432SJohn Baldwin 		memcpy(hmac_key, key, klen);
414c0341432SJohn Baldwin 
415c0341432SJohn Baldwin 	for (i = 0; i < axf->blocksize; i++)
416c0341432SJohn Baldwin 		hmac_key[i] ^= padval;
417c0341432SJohn Baldwin 
418c0341432SJohn Baldwin 	axf->Init(auth_ctx);
419c0341432SJohn Baldwin 	axf->Update(auth_ctx, hmac_key, axf->blocksize);
42017a831eaSJohn Baldwin 	explicit_bzero(hmac_key, sizeof(hmac_key));
421c0341432SJohn Baldwin }
422c0341432SJohn Baldwin 
423c0341432SJohn Baldwin void
424d588dc7dSMark Johnston hmac_init_ipad(const struct auth_hash *axf, const char *key, int klen,
425c0341432SJohn Baldwin     void *auth_ctx)
426c0341432SJohn Baldwin {
427c0341432SJohn Baldwin 
428c0341432SJohn Baldwin 	hmac_init_pad(axf, key, klen, auth_ctx, HMAC_IPAD_VAL);
429c0341432SJohn Baldwin }
430c0341432SJohn Baldwin 
431c0341432SJohn Baldwin void
432d588dc7dSMark Johnston hmac_init_opad(const struct auth_hash *axf, const char *key, int klen,
433c0341432SJohn Baldwin     void *auth_ctx)
434c0341432SJohn Baldwin {
435c0341432SJohn Baldwin 
436c0341432SJohn Baldwin 	hmac_init_pad(axf, key, klen, auth_ctx, HMAC_OPAD_VAL);
437c0341432SJohn Baldwin }
438c0341432SJohn Baldwin 
439c0341432SJohn Baldwin static void
44051e45326SSam Leffler crypto_destroy(void)
44151e45326SSam Leffler {
44239bbca6fSFabien Thomas 	struct crypto_ret_worker *ret_worker;
443c0341432SJohn Baldwin 	int i;
44439bbca6fSFabien Thomas 
44551e45326SSam Leffler 	/*
44651e45326SSam Leffler 	 * Terminate any crypto threads.
44751e45326SSam Leffler 	 */
44839bbca6fSFabien Thomas 	if (crypto_tq != NULL)
44939bbca6fSFabien Thomas 		taskqueue_drain_all(crypto_tq);
45051e45326SSam Leffler 	CRYPTO_DRIVER_LOCK();
45171785781SJohn Baldwin 	crypto_terminate(&cryptotd, &crp_q);
45239bbca6fSFabien Thomas 	FOREACH_CRYPTO_RETW(ret_worker)
45371785781SJohn Baldwin 		crypto_terminate(&ret_worker->td, &ret_worker->crp_ret_q);
45451e45326SSam Leffler 	CRYPTO_DRIVER_UNLOCK();
45551e45326SSam Leffler 
45651e45326SSam Leffler 	/* XXX flush queues??? */
45751e45326SSam Leffler 
45851e45326SSam Leffler 	/*
45951e45326SSam Leffler 	 * Reclaim dynamically allocated resources.
46051e45326SSam Leffler 	 */
461c0341432SJohn Baldwin 	for (i = 0; i < crypto_drivers_size; i++) {
462c0341432SJohn Baldwin 		if (crypto_drivers[i] != NULL)
463c0341432SJohn Baldwin 			cap_rele(crypto_drivers[i]);
464c0341432SJohn Baldwin 	}
46551e45326SSam Leffler 	free(crypto_drivers, M_CRYPTO_DATA);
46651e45326SSam Leffler 
46751e45326SSam Leffler 	if (cryptop_zone != NULL)
46851e45326SSam Leffler 		uma_zdestroy(cryptop_zone);
46951e45326SSam Leffler 	mtx_destroy(&crypto_q_mtx);
47039bbca6fSFabien Thomas 	FOREACH_CRYPTO_RETW(ret_worker)
47139bbca6fSFabien Thomas 		mtx_destroy(&ret_worker->crypto_ret_mtx);
47239bbca6fSFabien Thomas 	free(crypto_ret_workers, M_CRYPTO_DATA);
47339bbca6fSFabien Thomas 	if (crypto_tq != NULL)
47439bbca6fSFabien Thomas 		taskqueue_free(crypto_tq);
47551e45326SSam Leffler 	mtx_destroy(&crypto_drivers_mtx);
476091d81d1SSam Leffler }
477f544a528SMark Murray 
4781b0909d5SConrad Meyer uint32_t
4791b0909d5SConrad Meyer crypto_ses2hid(crypto_session_t crypto_session)
4801b0909d5SConrad Meyer {
481c0341432SJohn Baldwin 	return (crypto_session->cap->cc_hid);
4821b0909d5SConrad Meyer }
4831b0909d5SConrad Meyer 
4841b0909d5SConrad Meyer uint32_t
4851b0909d5SConrad Meyer crypto_ses2caps(crypto_session_t crypto_session)
4861b0909d5SConrad Meyer {
487c0341432SJohn Baldwin 	return (crypto_session->cap->cc_flags & 0xff000000);
4881b0909d5SConrad Meyer }
4891b0909d5SConrad Meyer 
4901b0909d5SConrad Meyer void *
4911b0909d5SConrad Meyer crypto_get_driver_session(crypto_session_t crypto_session)
4921b0909d5SConrad Meyer {
493d1816248SMark Johnston 	return (crypto_session + 1);
4941b0909d5SConrad Meyer }
4951b0909d5SConrad Meyer 
496c0341432SJohn Baldwin const struct crypto_session_params *
497c0341432SJohn Baldwin crypto_get_params(crypto_session_t crypto_session)
498c0341432SJohn Baldwin {
499c0341432SJohn Baldwin 	return (&crypto_session->csp);
500c0341432SJohn Baldwin }
501c0341432SJohn Baldwin 
502d8787d4fSMark Johnston const struct auth_hash *
503c0341432SJohn Baldwin crypto_auth_hash(const struct crypto_session_params *csp)
504c0341432SJohn Baldwin {
505c0341432SJohn Baldwin 
506c0341432SJohn Baldwin 	switch (csp->csp_auth_alg) {
507c0341432SJohn Baldwin 	case CRYPTO_SHA1_HMAC:
508c0341432SJohn Baldwin 		return (&auth_hash_hmac_sha1);
509c0341432SJohn Baldwin 	case CRYPTO_SHA2_224_HMAC:
510c0341432SJohn Baldwin 		return (&auth_hash_hmac_sha2_224);
511c0341432SJohn Baldwin 	case CRYPTO_SHA2_256_HMAC:
512c0341432SJohn Baldwin 		return (&auth_hash_hmac_sha2_256);
513c0341432SJohn Baldwin 	case CRYPTO_SHA2_384_HMAC:
514c0341432SJohn Baldwin 		return (&auth_hash_hmac_sha2_384);
515c0341432SJohn Baldwin 	case CRYPTO_SHA2_512_HMAC:
516c0341432SJohn Baldwin 		return (&auth_hash_hmac_sha2_512);
517c0341432SJohn Baldwin 	case CRYPTO_NULL_HMAC:
518c0341432SJohn Baldwin 		return (&auth_hash_null);
519c0341432SJohn Baldwin 	case CRYPTO_RIPEMD160_HMAC:
520c0341432SJohn Baldwin 		return (&auth_hash_hmac_ripemd_160);
521c3a688efSJohn Baldwin 	case CRYPTO_RIPEMD160:
522c3a688efSJohn Baldwin 		return (&auth_hash_ripemd_160);
523c0341432SJohn Baldwin 	case CRYPTO_SHA1:
524c0341432SJohn Baldwin 		return (&auth_hash_sha1);
525c0341432SJohn Baldwin 	case CRYPTO_SHA2_224:
526c0341432SJohn Baldwin 		return (&auth_hash_sha2_224);
527c0341432SJohn Baldwin 	case CRYPTO_SHA2_256:
528c0341432SJohn Baldwin 		return (&auth_hash_sha2_256);
529c0341432SJohn Baldwin 	case CRYPTO_SHA2_384:
530c0341432SJohn Baldwin 		return (&auth_hash_sha2_384);
531c0341432SJohn Baldwin 	case CRYPTO_SHA2_512:
532c0341432SJohn Baldwin 		return (&auth_hash_sha2_512);
533c0341432SJohn Baldwin 	case CRYPTO_AES_NIST_GMAC:
534c0341432SJohn Baldwin 		switch (csp->csp_auth_klen) {
535c0341432SJohn Baldwin 		case 128 / 8:
536c0341432SJohn Baldwin 			return (&auth_hash_nist_gmac_aes_128);
537c0341432SJohn Baldwin 		case 192 / 8:
538c0341432SJohn Baldwin 			return (&auth_hash_nist_gmac_aes_192);
539c0341432SJohn Baldwin 		case 256 / 8:
540c0341432SJohn Baldwin 			return (&auth_hash_nist_gmac_aes_256);
541c0341432SJohn Baldwin 		default:
542c0341432SJohn Baldwin 			return (NULL);
543c0341432SJohn Baldwin 		}
544c0341432SJohn Baldwin 	case CRYPTO_BLAKE2B:
545c0341432SJohn Baldwin 		return (&auth_hash_blake2b);
546c0341432SJohn Baldwin 	case CRYPTO_BLAKE2S:
547c0341432SJohn Baldwin 		return (&auth_hash_blake2s);
548c0341432SJohn Baldwin 	case CRYPTO_POLY1305:
549c0341432SJohn Baldwin 		return (&auth_hash_poly1305);
550c0341432SJohn Baldwin 	case CRYPTO_AES_CCM_CBC_MAC:
551c0341432SJohn Baldwin 		switch (csp->csp_auth_klen) {
552c0341432SJohn Baldwin 		case 128 / 8:
553c0341432SJohn Baldwin 			return (&auth_hash_ccm_cbc_mac_128);
554c0341432SJohn Baldwin 		case 192 / 8:
555c0341432SJohn Baldwin 			return (&auth_hash_ccm_cbc_mac_192);
556c0341432SJohn Baldwin 		case 256 / 8:
557c0341432SJohn Baldwin 			return (&auth_hash_ccm_cbc_mac_256);
558c0341432SJohn Baldwin 		default:
559c0341432SJohn Baldwin 			return (NULL);
560c0341432SJohn Baldwin 		}
561c0341432SJohn Baldwin 	default:
562c0341432SJohn Baldwin 		return (NULL);
563c0341432SJohn Baldwin 	}
564c0341432SJohn Baldwin }
565c0341432SJohn Baldwin 
566d8787d4fSMark Johnston const struct enc_xform *
567c0341432SJohn Baldwin crypto_cipher(const struct crypto_session_params *csp)
568c0341432SJohn Baldwin {
569c0341432SJohn Baldwin 
570c0341432SJohn Baldwin 	switch (csp->csp_cipher_alg) {
571246982c1SJohn Baldwin 	case CRYPTO_AES_CBC:
572246982c1SJohn Baldwin 		return (&enc_xform_aes_cbc);
573c0341432SJohn Baldwin 	case CRYPTO_AES_XTS:
574c0341432SJohn Baldwin 		return (&enc_xform_aes_xts);
575c0341432SJohn Baldwin 	case CRYPTO_AES_ICM:
576c0341432SJohn Baldwin 		return (&enc_xform_aes_icm);
577c0341432SJohn Baldwin 	case CRYPTO_AES_NIST_GCM_16:
578c0341432SJohn Baldwin 		return (&enc_xform_aes_nist_gcm);
579c0341432SJohn Baldwin 	case CRYPTO_CAMELLIA_CBC:
580c0341432SJohn Baldwin 		return (&enc_xform_camellia);
581c0341432SJohn Baldwin 	case CRYPTO_NULL_CBC:
582c0341432SJohn Baldwin 		return (&enc_xform_null);
583c0341432SJohn Baldwin 	case CRYPTO_CHACHA20:
584c0341432SJohn Baldwin 		return (&enc_xform_chacha20);
585c0341432SJohn Baldwin 	case CRYPTO_AES_CCM_16:
586c0341432SJohn Baldwin 		return (&enc_xform_ccm);
587fc8fc743SJohn Baldwin 	case CRYPTO_CHACHA20_POLY1305:
588fc8fc743SJohn Baldwin 		return (&enc_xform_chacha20_poly1305);
5898f35841fSJohn Baldwin 	case CRYPTO_XCHACHA20_POLY1305:
5908f35841fSJohn Baldwin 		return (&enc_xform_xchacha20_poly1305);
591c0341432SJohn Baldwin 	default:
592c0341432SJohn Baldwin 		return (NULL);
593c0341432SJohn Baldwin 	}
594c0341432SJohn Baldwin }
595c0341432SJohn Baldwin 
5966810ad6fSSam Leffler static struct cryptocap *
597d3d79e96SJohn Baldwin crypto_checkdriver(uint32_t hid)
5986810ad6fSSam Leffler {
5996810ad6fSSam Leffler 
600c0341432SJohn Baldwin 	return (hid >= crypto_drivers_size ? NULL : crypto_drivers[hid]);
601f544a528SMark Murray }
602f544a528SMark Murray 
603091d81d1SSam Leffler /*
6046810ad6fSSam Leffler  * Select a driver for a new session that supports the specified
6056810ad6fSSam Leffler  * algorithms and, optionally, is constrained according to the flags.
606091d81d1SSam Leffler  */
6076810ad6fSSam Leffler static struct cryptocap *
608c0341432SJohn Baldwin crypto_select_driver(const struct crypto_session_params *csp, int flags)
6096810ad6fSSam Leffler {
6106810ad6fSSam Leffler 	struct cryptocap *cap, *best;
611c0341432SJohn Baldwin 	int best_match, error, hid;
6126810ad6fSSam Leffler 
6136810ad6fSSam Leffler 	CRYPTO_DRIVER_ASSERT();
614091d81d1SSam Leffler 
6156810ad6fSSam Leffler 	best = NULL;
616c0341432SJohn Baldwin 	for (hid = 0; hid < crypto_drivers_size; hid++) {
617091d81d1SSam Leffler 		/*
618c0341432SJohn Baldwin 		 * If there is no driver for this slot, or the driver
619c0341432SJohn Baldwin 		 * is not appropriate (hardware or software based on
620c0341432SJohn Baldwin 		 * match), then skip.
621091d81d1SSam Leffler 		 */
622c0341432SJohn Baldwin 		cap = crypto_drivers[hid];
623c0341432SJohn Baldwin 		if (cap == NULL ||
624c0341432SJohn Baldwin 		    (cap->cc_flags & flags) == 0)
625091d81d1SSam Leffler 			continue;
626091d81d1SSam Leffler 
627c0341432SJohn Baldwin 		error = CRYPTODEV_PROBESESSION(cap->cc_dev, csp);
628c0341432SJohn Baldwin 		if (error >= 0)
629c0341432SJohn Baldwin 			continue;
630c0341432SJohn Baldwin 
631c0341432SJohn Baldwin 		/*
632c0341432SJohn Baldwin 		 * Use the driver with the highest probe value.
633c0341432SJohn Baldwin 		 * Hardware drivers use a higher probe value than
634c0341432SJohn Baldwin 		 * software.  In case of a tie, prefer the driver with
635c0341432SJohn Baldwin 		 * the fewest active sessions.
636c0341432SJohn Baldwin 		 */
637c0341432SJohn Baldwin 		if (best == NULL || error > best_match ||
638c0341432SJohn Baldwin 		    (error == best_match &&
639c0341432SJohn Baldwin 		    cap->cc_sessions < best->cc_sessions)) {
6406810ad6fSSam Leffler 			best = cap;
641c0341432SJohn Baldwin 			best_match = error;
6426810ad6fSSam Leffler 		}
6436810ad6fSSam Leffler 	}
6446810ad6fSSam Leffler 	return best;
6456810ad6fSSam Leffler }
646091d81d1SSam Leffler 
647ad557055SJohn Baldwin static enum alg_type {
648ad557055SJohn Baldwin 	ALG_NONE = 0,
649ad557055SJohn Baldwin 	ALG_CIPHER,
650ad557055SJohn Baldwin 	ALG_DIGEST,
651ad557055SJohn Baldwin 	ALG_KEYED_DIGEST,
652ad557055SJohn Baldwin 	ALG_COMPRESSION,
653ad557055SJohn Baldwin 	ALG_AEAD
654ad557055SJohn Baldwin } alg_types[] = {
655ad557055SJohn Baldwin 	[CRYPTO_SHA1_HMAC] = ALG_KEYED_DIGEST,
656ad557055SJohn Baldwin 	[CRYPTO_RIPEMD160_HMAC] = ALG_KEYED_DIGEST,
657ad557055SJohn Baldwin 	[CRYPTO_AES_CBC] = ALG_CIPHER,
658ad557055SJohn Baldwin 	[CRYPTO_SHA1] = ALG_DIGEST,
659ad557055SJohn Baldwin 	[CRYPTO_NULL_HMAC] = ALG_DIGEST,
660ad557055SJohn Baldwin 	[CRYPTO_NULL_CBC] = ALG_CIPHER,
661ad557055SJohn Baldwin 	[CRYPTO_DEFLATE_COMP] = ALG_COMPRESSION,
662ad557055SJohn Baldwin 	[CRYPTO_SHA2_256_HMAC] = ALG_KEYED_DIGEST,
663ad557055SJohn Baldwin 	[CRYPTO_SHA2_384_HMAC] = ALG_KEYED_DIGEST,
664ad557055SJohn Baldwin 	[CRYPTO_SHA2_512_HMAC] = ALG_KEYED_DIGEST,
665ad557055SJohn Baldwin 	[CRYPTO_CAMELLIA_CBC] = ALG_CIPHER,
666ad557055SJohn Baldwin 	[CRYPTO_AES_XTS] = ALG_CIPHER,
667ad557055SJohn Baldwin 	[CRYPTO_AES_ICM] = ALG_CIPHER,
668ad557055SJohn Baldwin 	[CRYPTO_AES_NIST_GMAC] = ALG_KEYED_DIGEST,
669ad557055SJohn Baldwin 	[CRYPTO_AES_NIST_GCM_16] = ALG_AEAD,
670ad557055SJohn Baldwin 	[CRYPTO_BLAKE2B] = ALG_KEYED_DIGEST,
671ad557055SJohn Baldwin 	[CRYPTO_BLAKE2S] = ALG_KEYED_DIGEST,
672ad557055SJohn Baldwin 	[CRYPTO_CHACHA20] = ALG_CIPHER,
673ad557055SJohn Baldwin 	[CRYPTO_SHA2_224_HMAC] = ALG_KEYED_DIGEST,
674ad557055SJohn Baldwin 	[CRYPTO_RIPEMD160] = ALG_DIGEST,
675ad557055SJohn Baldwin 	[CRYPTO_SHA2_224] = ALG_DIGEST,
676ad557055SJohn Baldwin 	[CRYPTO_SHA2_256] = ALG_DIGEST,
677ad557055SJohn Baldwin 	[CRYPTO_SHA2_384] = ALG_DIGEST,
678ad557055SJohn Baldwin 	[CRYPTO_SHA2_512] = ALG_DIGEST,
679ad557055SJohn Baldwin 	[CRYPTO_POLY1305] = ALG_KEYED_DIGEST,
680ad557055SJohn Baldwin 	[CRYPTO_AES_CCM_CBC_MAC] = ALG_KEYED_DIGEST,
681ad557055SJohn Baldwin 	[CRYPTO_AES_CCM_16] = ALG_AEAD,
682fc8fc743SJohn Baldwin 	[CRYPTO_CHACHA20_POLY1305] = ALG_AEAD,
6838f35841fSJohn Baldwin 	[CRYPTO_XCHACHA20_POLY1305] = ALG_AEAD,
684ad557055SJohn Baldwin };
685ad557055SJohn Baldwin 
686ad557055SJohn Baldwin static enum alg_type
687ad557055SJohn Baldwin alg_type(int alg)
688ad557055SJohn Baldwin {
689ad557055SJohn Baldwin 
690ad557055SJohn Baldwin 	if (alg < nitems(alg_types))
691ad557055SJohn Baldwin 		return (alg_types[alg]);
692ad557055SJohn Baldwin 	return (ALG_NONE);
693ad557055SJohn Baldwin }
694ad557055SJohn Baldwin 
695c0341432SJohn Baldwin static bool
696c0341432SJohn Baldwin alg_is_compression(int alg)
697c0341432SJohn Baldwin {
698c0341432SJohn Baldwin 
699ad557055SJohn Baldwin 	return (alg_type(alg) == ALG_COMPRESSION);
700c0341432SJohn Baldwin }
701c0341432SJohn Baldwin 
702c0341432SJohn Baldwin static bool
703c0341432SJohn Baldwin alg_is_cipher(int alg)
704c0341432SJohn Baldwin {
705c0341432SJohn Baldwin 
706ad557055SJohn Baldwin 	return (alg_type(alg) == ALG_CIPHER);
707c0341432SJohn Baldwin }
708c0341432SJohn Baldwin 
709c0341432SJohn Baldwin static bool
710c0341432SJohn Baldwin alg_is_digest(int alg)
711c0341432SJohn Baldwin {
712c0341432SJohn Baldwin 
713ad557055SJohn Baldwin 	return (alg_type(alg) == ALG_DIGEST ||
714ad557055SJohn Baldwin 	    alg_type(alg) == ALG_KEYED_DIGEST);
715c0341432SJohn Baldwin }
716c0341432SJohn Baldwin 
717c0341432SJohn Baldwin static bool
718c0341432SJohn Baldwin alg_is_keyed_digest(int alg)
719c0341432SJohn Baldwin {
720c0341432SJohn Baldwin 
721ad557055SJohn Baldwin 	return (alg_type(alg) == ALG_KEYED_DIGEST);
722c0341432SJohn Baldwin }
723c0341432SJohn Baldwin 
724c0341432SJohn Baldwin static bool
725c0341432SJohn Baldwin alg_is_aead(int alg)
726c0341432SJohn Baldwin {
727c0341432SJohn Baldwin 
728ad557055SJohn Baldwin 	return (alg_type(alg) == ALG_AEAD);
729c0341432SJohn Baldwin }
730c0341432SJohn Baldwin 
731ae18720dSJohn Baldwin static bool
732ae18720dSJohn Baldwin ccm_tag_length_valid(int len)
733ae18720dSJohn Baldwin {
734ae18720dSJohn Baldwin 	/* RFC 3610 */
735ae18720dSJohn Baldwin 	switch (len) {
736ae18720dSJohn Baldwin 	case 4:
737ae18720dSJohn Baldwin 	case 6:
738ae18720dSJohn Baldwin 	case 8:
739ae18720dSJohn Baldwin 	case 10:
740ae18720dSJohn Baldwin 	case 12:
741ae18720dSJohn Baldwin 	case 14:
742ae18720dSJohn Baldwin 	case 16:
743ae18720dSJohn Baldwin 		return (true);
744ae18720dSJohn Baldwin 	default:
745ae18720dSJohn Baldwin 		return (false);
746ae18720dSJohn Baldwin 	}
747ae18720dSJohn Baldwin }
748ae18720dSJohn Baldwin 
7497e89ae49SMarcin Wojtas #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN)
7507e89ae49SMarcin Wojtas 
751c0341432SJohn Baldwin /* Various sanity checks on crypto session parameters. */
752c0341432SJohn Baldwin static bool
753c0341432SJohn Baldwin check_csp(const struct crypto_session_params *csp)
754c0341432SJohn Baldwin {
755d8787d4fSMark Johnston 	const struct auth_hash *axf;
756c0341432SJohn Baldwin 
757c0341432SJohn Baldwin 	/* Mode-independent checks. */
7587e89ae49SMarcin Wojtas 	if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
759c0341432SJohn Baldwin 		return (false);
760c0341432SJohn Baldwin 	if (csp->csp_ivlen < 0 || csp->csp_cipher_klen < 0 ||
761c0341432SJohn Baldwin 	    csp->csp_auth_klen < 0 || csp->csp_auth_mlen < 0)
762c0341432SJohn Baldwin 		return (false);
763c0341432SJohn Baldwin 	if (csp->csp_auth_key != NULL && csp->csp_auth_klen == 0)
764c0341432SJohn Baldwin 		return (false);
765c0341432SJohn Baldwin 	if (csp->csp_cipher_key != NULL && csp->csp_cipher_klen == 0)
766c0341432SJohn Baldwin 		return (false);
767c0341432SJohn Baldwin 
768c0341432SJohn Baldwin 	switch (csp->csp_mode) {
769c0341432SJohn Baldwin 	case CSP_MODE_COMPRESS:
770c0341432SJohn Baldwin 		if (!alg_is_compression(csp->csp_cipher_alg))
771c0341432SJohn Baldwin 			return (false);
7729c0e3d3aSJohn Baldwin 		if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT)
773c0341432SJohn Baldwin 			return (false);
7749b774dc0SJohn Baldwin 		if (csp->csp_flags & CSP_F_SEPARATE_AAD)
7759b774dc0SJohn Baldwin 			return (false);
776c0341432SJohn Baldwin 		if (csp->csp_cipher_klen != 0 || csp->csp_ivlen != 0 ||
777c0341432SJohn Baldwin 		    csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
778c0341432SJohn Baldwin 		    csp->csp_auth_mlen != 0)
779c0341432SJohn Baldwin 			return (false);
780c0341432SJohn Baldwin 		break;
781c0341432SJohn Baldwin 	case CSP_MODE_CIPHER:
782c0341432SJohn Baldwin 		if (!alg_is_cipher(csp->csp_cipher_alg))
783c0341432SJohn Baldwin 			return (false);
7849b774dc0SJohn Baldwin 		if (csp->csp_flags & CSP_F_SEPARATE_AAD)
7859b774dc0SJohn Baldwin 			return (false);
786c0341432SJohn Baldwin 		if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
787c0341432SJohn Baldwin 			if (csp->csp_cipher_klen == 0)
788c0341432SJohn Baldwin 				return (false);
789c0341432SJohn Baldwin 			if (csp->csp_ivlen == 0)
790c0341432SJohn Baldwin 				return (false);
791c0341432SJohn Baldwin 		}
792c0341432SJohn Baldwin 		if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
793c0341432SJohn Baldwin 			return (false);
794c0341432SJohn Baldwin 		if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
795c0341432SJohn Baldwin 		    csp->csp_auth_mlen != 0)
796c0341432SJohn Baldwin 			return (false);
797c0341432SJohn Baldwin 		break;
798c0341432SJohn Baldwin 	case CSP_MODE_DIGEST:
799c0341432SJohn Baldwin 		if (csp->csp_cipher_alg != 0 || csp->csp_cipher_klen != 0)
800c0341432SJohn Baldwin 			return (false);
801c0341432SJohn Baldwin 
8029b774dc0SJohn Baldwin 		if (csp->csp_flags & CSP_F_SEPARATE_AAD)
8039b774dc0SJohn Baldwin 			return (false);
8049b774dc0SJohn Baldwin 
805c0341432SJohn Baldwin 		/* IV is optional for digests (e.g. GMAC). */
806ae18720dSJohn Baldwin 		switch (csp->csp_auth_alg) {
807ae18720dSJohn Baldwin 		case CRYPTO_AES_CCM_CBC_MAC:
808ae18720dSJohn Baldwin 			if (csp->csp_ivlen < 7 || csp->csp_ivlen > 13)
809c0341432SJohn Baldwin 				return (false);
810ae18720dSJohn Baldwin 			break;
811ae18720dSJohn Baldwin 		case CRYPTO_AES_NIST_GMAC:
812ae18720dSJohn Baldwin 			if (csp->csp_ivlen != AES_GCM_IV_LEN)
813ae18720dSJohn Baldwin 				return (false);
814ae18720dSJohn Baldwin 			break;
815ae18720dSJohn Baldwin 		default:
816ae18720dSJohn Baldwin 			if (csp->csp_ivlen != 0)
817ae18720dSJohn Baldwin 				return (false);
818ae18720dSJohn Baldwin 			break;
819ae18720dSJohn Baldwin 		}
820ae18720dSJohn Baldwin 
821c0341432SJohn Baldwin 		if (!alg_is_digest(csp->csp_auth_alg))
822c0341432SJohn Baldwin 			return (false);
823c0341432SJohn Baldwin 
824c0341432SJohn Baldwin 		/* Key is optional for BLAKE2 digests. */
825c0341432SJohn Baldwin 		if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
826c0341432SJohn Baldwin 		    csp->csp_auth_alg == CRYPTO_BLAKE2S)
827c0341432SJohn Baldwin 			;
828c0341432SJohn Baldwin 		else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
829c0341432SJohn Baldwin 			if (csp->csp_auth_klen == 0)
830c0341432SJohn Baldwin 				return (false);
831c0341432SJohn Baldwin 		} else {
832c0341432SJohn Baldwin 			if (csp->csp_auth_klen != 0)
833c0341432SJohn Baldwin 				return (false);
834c0341432SJohn Baldwin 		}
835c0341432SJohn Baldwin 		if (csp->csp_auth_mlen != 0) {
836c0341432SJohn Baldwin 			axf = crypto_auth_hash(csp);
837c0341432SJohn Baldwin 			if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
838c0341432SJohn Baldwin 				return (false);
839ae18720dSJohn Baldwin 
840ae18720dSJohn Baldwin 			if (csp->csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC &&
841ae18720dSJohn Baldwin 			    !ccm_tag_length_valid(csp->csp_auth_mlen))
842ae18720dSJohn Baldwin 				return (false);
843c0341432SJohn Baldwin 		}
844c0341432SJohn Baldwin 		break;
845c0341432SJohn Baldwin 	case CSP_MODE_AEAD:
846c0341432SJohn Baldwin 		if (!alg_is_aead(csp->csp_cipher_alg))
847c0341432SJohn Baldwin 			return (false);
848c0341432SJohn Baldwin 		if (csp->csp_cipher_klen == 0)
849c0341432SJohn Baldwin 			return (false);
850c0341432SJohn Baldwin 		if (csp->csp_ivlen == 0 ||
851c0341432SJohn Baldwin 		    csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
852c0341432SJohn Baldwin 			return (false);
853c0341432SJohn Baldwin 		if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0)
854c0341432SJohn Baldwin 			return (false);
855c0341432SJohn Baldwin 
856c0341432SJohn Baldwin 		switch (csp->csp_cipher_alg) {
857c0341432SJohn Baldwin 		case CRYPTO_AES_CCM_16:
858ae18720dSJohn Baldwin 			if (csp->csp_auth_mlen != 0 &&
859ae18720dSJohn Baldwin 			    !ccm_tag_length_valid(csp->csp_auth_mlen))
860ae18720dSJohn Baldwin 				return (false);
861ae18720dSJohn Baldwin 
862ae18720dSJohn Baldwin 			if (csp->csp_ivlen < 7 || csp->csp_ivlen > 13)
863ae18720dSJohn Baldwin 				return (false);
864ae18720dSJohn Baldwin 			break;
865ae18720dSJohn Baldwin 		case CRYPTO_AES_NIST_GCM_16:
8666e17a2e0SJohn Baldwin 			if (csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
8676e17a2e0SJohn Baldwin 				return (false);
8686e17a2e0SJohn Baldwin 
8696e17a2e0SJohn Baldwin 			if (csp->csp_ivlen != AES_GCM_IV_LEN)
870c0341432SJohn Baldwin 				return (false);
871c0341432SJohn Baldwin 			break;
87242dcd395SJohn Baldwin 		case CRYPTO_CHACHA20_POLY1305:
87342dcd395SJohn Baldwin 			if (csp->csp_ivlen != 8 && csp->csp_ivlen != 12)
87442dcd395SJohn Baldwin 				return (false);
87542dcd395SJohn Baldwin 			if (csp->csp_auth_mlen > POLY1305_HASH_LEN)
87642dcd395SJohn Baldwin 				return (false);
87742dcd395SJohn Baldwin 			break;
8788f35841fSJohn Baldwin 		case CRYPTO_XCHACHA20_POLY1305:
8798f35841fSJohn Baldwin 			if (csp->csp_ivlen != XCHACHA20_POLY1305_IV_LEN)
8808f35841fSJohn Baldwin 				return (false);
8818f35841fSJohn Baldwin 			if (csp->csp_auth_mlen > POLY1305_HASH_LEN)
8828f35841fSJohn Baldwin 				return (false);
8838f35841fSJohn Baldwin 			break;
884c0341432SJohn Baldwin 		}
885c0341432SJohn Baldwin 		break;
886c0341432SJohn Baldwin 	case CSP_MODE_ETA:
887c0341432SJohn Baldwin 		if (!alg_is_cipher(csp->csp_cipher_alg))
888c0341432SJohn Baldwin 			return (false);
889c0341432SJohn Baldwin 		if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
890c0341432SJohn Baldwin 			if (csp->csp_cipher_klen == 0)
891c0341432SJohn Baldwin 				return (false);
892c0341432SJohn Baldwin 			if (csp->csp_ivlen == 0)
893c0341432SJohn Baldwin 				return (false);
894c0341432SJohn Baldwin 		}
895c0341432SJohn Baldwin 		if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
896c0341432SJohn Baldwin 			return (false);
897c0341432SJohn Baldwin 		if (!alg_is_digest(csp->csp_auth_alg))
898c0341432SJohn Baldwin 			return (false);
899c0341432SJohn Baldwin 
900c0341432SJohn Baldwin 		/* Key is optional for BLAKE2 digests. */
901c0341432SJohn Baldwin 		if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
902c0341432SJohn Baldwin 		    csp->csp_auth_alg == CRYPTO_BLAKE2S)
903c0341432SJohn Baldwin 			;
904c0341432SJohn Baldwin 		else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
905c0341432SJohn Baldwin 			if (csp->csp_auth_klen == 0)
906c0341432SJohn Baldwin 				return (false);
907c0341432SJohn Baldwin 		} else {
908c0341432SJohn Baldwin 			if (csp->csp_auth_klen != 0)
909c0341432SJohn Baldwin 				return (false);
910c0341432SJohn Baldwin 		}
911c0341432SJohn Baldwin 		if (csp->csp_auth_mlen != 0) {
912c0341432SJohn Baldwin 			axf = crypto_auth_hash(csp);
913c0341432SJohn Baldwin 			if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
914c0341432SJohn Baldwin 				return (false);
915c0341432SJohn Baldwin 		}
916c0341432SJohn Baldwin 		break;
917c0341432SJohn Baldwin 	default:
918c0341432SJohn Baldwin 		return (false);
919c0341432SJohn Baldwin 	}
920c0341432SJohn Baldwin 
921c0341432SJohn Baldwin 	return (true);
922c0341432SJohn Baldwin }
923c0341432SJohn Baldwin 
924c0341432SJohn Baldwin /*
925c0341432SJohn Baldwin  * Delete a session after it has been detached from its driver.
926c0341432SJohn Baldwin  */
927c0341432SJohn Baldwin static void
928c0341432SJohn Baldwin crypto_deletesession(crypto_session_t cses)
929c0341432SJohn Baldwin {
930c0341432SJohn Baldwin 	struct cryptocap *cap;
931c0341432SJohn Baldwin 
932c0341432SJohn Baldwin 	cap = cses->cap;
933c0341432SJohn Baldwin 
934d1816248SMark Johnston 	zfree(cses, M_CRYPTO_DATA);
935c0341432SJohn Baldwin 
936c0341432SJohn Baldwin 	CRYPTO_DRIVER_LOCK();
937c0341432SJohn Baldwin 	cap->cc_sessions--;
938c0341432SJohn Baldwin 	if (cap->cc_sessions == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP)
939c0341432SJohn Baldwin 		wakeup(cap);
940c0341432SJohn Baldwin 	CRYPTO_DRIVER_UNLOCK();
941c0341432SJohn Baldwin 	cap_rele(cap);
942c0341432SJohn Baldwin }
943c0341432SJohn Baldwin 
944694e0113SPawel Jakub Dawidek /*
9456810ad6fSSam Leffler  * Create a new session.  The crid argument specifies a crypto
9466810ad6fSSam Leffler  * driver to use or constraints on a driver to select (hardware
9476810ad6fSSam Leffler  * only, software only, either).  Whatever driver is selected
9486810ad6fSSam Leffler  * must be capable of the requested crypto algorithms.
949694e0113SPawel Jakub Dawidek  */
9506810ad6fSSam Leffler int
951c0341432SJohn Baldwin crypto_newsession(crypto_session_t *cses,
952c0341432SJohn Baldwin     const struct crypto_session_params *csp, int crid)
9536810ad6fSSam Leffler {
95498d788c8SMark Johnston 	static uint64_t sessid = 0;
9551b0909d5SConrad Meyer 	crypto_session_t res;
9566810ad6fSSam Leffler 	struct cryptocap *cap;
9576810ad6fSSam Leffler 	int err;
9586810ad6fSSam Leffler 
959c0341432SJohn Baldwin 	if (!check_csp(csp))
960c0341432SJohn Baldwin 		return (EINVAL);
961c0341432SJohn Baldwin 
9621b0909d5SConrad Meyer 	res = NULL;
9631b0909d5SConrad Meyer 
9646810ad6fSSam Leffler 	CRYPTO_DRIVER_LOCK();
9656810ad6fSSam Leffler 	if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
966694e0113SPawel Jakub Dawidek 		/*
9676810ad6fSSam Leffler 		 * Use specified driver; verify it is capable.
968694e0113SPawel Jakub Dawidek 		 */
9696810ad6fSSam Leffler 		cap = crypto_checkdriver(crid);
970c0341432SJohn Baldwin 		if (cap != NULL && CRYPTODEV_PROBESESSION(cap->cc_dev, csp) > 0)
971694e0113SPawel Jakub Dawidek 			cap = NULL;
9726810ad6fSSam Leffler 	} else {
9736810ad6fSSam Leffler 		/*
9746810ad6fSSam Leffler 		 * No requested driver; select based on crid flags.
9756810ad6fSSam Leffler 		 */
976c0341432SJohn Baldwin 		cap = crypto_select_driver(csp, crid);
977694e0113SPawel Jakub Dawidek 	}
9781b0909d5SConrad Meyer 	if (cap == NULL) {
979c0341432SJohn Baldwin 		CRYPTO_DRIVER_UNLOCK();
98008fca7a5SJohn-Mark Gurney 		CRYPTDEB("no driver");
981c0341432SJohn Baldwin 		return (EOPNOTSUPP);
98208fca7a5SJohn-Mark Gurney 	}
983c0341432SJohn Baldwin 	cap_ref(cap);
9841b0909d5SConrad Meyer 	cap->cc_sessions++;
985091d81d1SSam Leffler 	CRYPTO_DRIVER_UNLOCK();
9861b0909d5SConrad Meyer 
9878adcc757SMark Johnston 	/* Allocate a single block for the generic session and driver softc. */
988d1816248SMark Johnston 	res = malloc(sizeof(*res) + cap->cc_session_size, M_CRYPTO_DATA,
989d1816248SMark Johnston 	    M_WAITOK | M_ZERO);
990c0341432SJohn Baldwin 	res->cap = cap;
991c0341432SJohn Baldwin 	res->csp = *csp;
99298d788c8SMark Johnston 	res->id = atomic_fetchadd_64(&sessid, 1);
9931b0909d5SConrad Meyer 
9941b0909d5SConrad Meyer 	/* Call the driver initialization routine. */
995c0341432SJohn Baldwin 	err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, csp);
9961b0909d5SConrad Meyer 	if (err != 0) {
9971b0909d5SConrad Meyer 		CRYPTDEB("dev newsession failed: %d", err);
998c0341432SJohn Baldwin 		crypto_deletesession(res);
999c0341432SJohn Baldwin 		return (err);
10001b0909d5SConrad Meyer 	}
10011b0909d5SConrad Meyer 
10021b0909d5SConrad Meyer 	*cses = res;
1003c0341432SJohn Baldwin 	return (0);
10044acae0acSPawel Jakub Dawidek }
10054acae0acSPawel Jakub Dawidek 
1006091d81d1SSam Leffler /*
1007091d81d1SSam Leffler  * Delete an existing session (or a reserved session on an unregistered
1008091d81d1SSam Leffler  * driver).
1009091d81d1SSam Leffler  */
10101b0909d5SConrad Meyer void
10111b0909d5SConrad Meyer crypto_freesession(crypto_session_t cses)
1012091d81d1SSam Leffler {
10134acae0acSPawel Jakub Dawidek 	struct cryptocap *cap;
10141b0909d5SConrad Meyer 
10151b0909d5SConrad Meyer 	if (cses == NULL)
10161b0909d5SConrad Meyer 		return;
1017091d81d1SSam Leffler 
1018c0341432SJohn Baldwin 	cap = cses->cap;
1019091d81d1SSam Leffler 
1020091d81d1SSam Leffler 	/* Call the driver cleanup routine, if available. */
10211b0909d5SConrad Meyer 	CRYPTODEV_FREESESSION(cap->cc_dev, cses);
10221b0909d5SConrad Meyer 
1023c0341432SJohn Baldwin 	crypto_deletesession(cses);
1024091d81d1SSam Leffler }
1025091d81d1SSam Leffler 
1026091d81d1SSam Leffler /*
1027c0341432SJohn Baldwin  * Return a new driver id.  Registers a driver with the system so that
1028c0341432SJohn Baldwin  * it can be probed by subsequent sessions.
1029091d81d1SSam Leffler  */
1030091d81d1SSam Leffler int32_t
10311b0909d5SConrad Meyer crypto_get_driverid(device_t dev, size_t sessionsize, int flags)
1032091d81d1SSam Leffler {
1033c0341432SJohn Baldwin 	struct cryptocap *cap, **newdrv;
1034091d81d1SSam Leffler 	int i;
1035091d81d1SSam Leffler 
10366810ad6fSSam Leffler 	if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
1037c0341432SJohn Baldwin 		device_printf(dev,
1038c0341432SJohn Baldwin 		    "no flags specified when registering driver\n");
10396810ad6fSSam Leffler 		return -1;
10406810ad6fSSam Leffler 	}
10416810ad6fSSam Leffler 
1042c0341432SJohn Baldwin 	cap = malloc(sizeof(*cap), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
1043c0341432SJohn Baldwin 	cap->cc_dev = dev;
1044c0341432SJohn Baldwin 	cap->cc_session_size = sessionsize;
1045c0341432SJohn Baldwin 	cap->cc_flags = flags;
1046c0341432SJohn Baldwin 	refcount_init(&cap->cc_refs, 1);
1047c0341432SJohn Baldwin 
1048091d81d1SSam Leffler 	CRYPTO_DRIVER_LOCK();
1049c0341432SJohn Baldwin 	for (;;) {
1050c0341432SJohn Baldwin 		for (i = 0; i < crypto_drivers_size; i++) {
1051c0341432SJohn Baldwin 			if (crypto_drivers[i] == NULL)
1052091d81d1SSam Leffler 				break;
10534acae0acSPawel Jakub Dawidek 		}
1054c0341432SJohn Baldwin 
1055c0341432SJohn Baldwin 		if (i < crypto_drivers_size)
1056c0341432SJohn Baldwin 			break;
1057091d81d1SSam Leffler 
1058091d81d1SSam Leffler 		/* Out of entries, allocate some more. */
1059c0341432SJohn Baldwin 
1060c0341432SJohn Baldwin 		if (2 * crypto_drivers_size <= crypto_drivers_size) {
1061091d81d1SSam Leffler 			CRYPTO_DRIVER_UNLOCK();
1062091d81d1SSam Leffler 			printf("crypto: driver count wraparound!\n");
1063c0341432SJohn Baldwin 			cap_rele(cap);
1064c0341432SJohn Baldwin 			return (-1);
1065091d81d1SSam Leffler 		}
1066091d81d1SSam Leffler 		CRYPTO_DRIVER_UNLOCK();
1067091d81d1SSam Leffler 
1068c0341432SJohn Baldwin 		newdrv = malloc(2 * crypto_drivers_size *
1069c0341432SJohn Baldwin 		    sizeof(*crypto_drivers), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
1070091d81d1SSam Leffler 
1071c0341432SJohn Baldwin 		CRYPTO_DRIVER_LOCK();
1072c0341432SJohn Baldwin 		memcpy(newdrv, crypto_drivers,
1073c0341432SJohn Baldwin 		    crypto_drivers_size * sizeof(*crypto_drivers));
1074c0341432SJohn Baldwin 
1075c0341432SJohn Baldwin 		crypto_drivers_size *= 2;
1076091d81d1SSam Leffler 
1077091d81d1SSam Leffler 		free(crypto_drivers, M_CRYPTO_DATA);
1078091d81d1SSam Leffler 		crypto_drivers = newdrv;
1079091d81d1SSam Leffler 	}
1080091d81d1SSam Leffler 
1081c0341432SJohn Baldwin 	cap->cc_hid = i;
1082c0341432SJohn Baldwin 	crypto_drivers[i] = cap;
1083c0341432SJohn Baldwin 	CRYPTO_DRIVER_UNLOCK();
1084c0341432SJohn Baldwin 
1085091d81d1SSam Leffler 	if (bootverbose)
1086d7d2f0d4SConrad Meyer 		printf("crypto: assign %s driver id %u, flags 0x%x\n",
10876810ad6fSSam Leffler 		    device_get_nameunit(dev), i, flags);
1088091d81d1SSam Leffler 
1089091d81d1SSam Leffler 	return i;
1090091d81d1SSam Leffler }
1091091d81d1SSam Leffler 
10926810ad6fSSam Leffler /*
10936810ad6fSSam Leffler  * Lookup a driver by name.  We match against the full device
10946810ad6fSSam Leffler  * name and unit, and against just the name.  The latter gives
10956810ad6fSSam Leffler  * us a simple widlcarding by device name.  On success return the
10966810ad6fSSam Leffler  * driver/hardware identifier; otherwise return -1.
10976810ad6fSSam Leffler  */
10986810ad6fSSam Leffler int
10996810ad6fSSam Leffler crypto_find_driver(const char *match)
1100091d81d1SSam Leffler {
1101c0341432SJohn Baldwin 	struct cryptocap *cap;
11026810ad6fSSam Leffler 	int i, len = strlen(match);
11036810ad6fSSam Leffler 
11046810ad6fSSam Leffler 	CRYPTO_DRIVER_LOCK();
1105c0341432SJohn Baldwin 	for (i = 0; i < crypto_drivers_size; i++) {
1106c0341432SJohn Baldwin 		if (crypto_drivers[i] == NULL)
11076810ad6fSSam Leffler 			continue;
1108c0341432SJohn Baldwin 		cap = crypto_drivers[i];
1109c0341432SJohn Baldwin 		if (strncmp(match, device_get_nameunit(cap->cc_dev), len) == 0 ||
1110c0341432SJohn Baldwin 		    strncmp(match, device_get_name(cap->cc_dev), len) == 0) {
1111c0341432SJohn Baldwin 			CRYPTO_DRIVER_UNLOCK();
1112c0341432SJohn Baldwin 			return (i);
1113c0341432SJohn Baldwin 		}
11146810ad6fSSam Leffler 	}
11156810ad6fSSam Leffler 	CRYPTO_DRIVER_UNLOCK();
1116c0341432SJohn Baldwin 	return (-1);
11176810ad6fSSam Leffler }
11186810ad6fSSam Leffler 
11196810ad6fSSam Leffler /*
11206810ad6fSSam Leffler  * Return the device_t for the specified driver or NULL
11216810ad6fSSam Leffler  * if the driver identifier is invalid.
11226810ad6fSSam Leffler  */
11236810ad6fSSam Leffler device_t
11246810ad6fSSam Leffler crypto_find_device_byhid(int hid)
11256810ad6fSSam Leffler {
1126c0341432SJohn Baldwin 	struct cryptocap *cap;
1127c0341432SJohn Baldwin 	device_t dev;
1128c0341432SJohn Baldwin 
1129c0341432SJohn Baldwin 	dev = NULL;
1130c0341432SJohn Baldwin 	CRYPTO_DRIVER_LOCK();
1131c0341432SJohn Baldwin 	cap = crypto_checkdriver(hid);
1132c0341432SJohn Baldwin 	if (cap != NULL)
1133c0341432SJohn Baldwin 		dev = cap->cc_dev;
1134c0341432SJohn Baldwin 	CRYPTO_DRIVER_UNLOCK();
1135c0341432SJohn Baldwin 	return (dev);
11366810ad6fSSam Leffler }
11376810ad6fSSam Leffler 
11386810ad6fSSam Leffler /*
11396810ad6fSSam Leffler  * Return the device/driver capabilities.
11406810ad6fSSam Leffler  */
11416810ad6fSSam Leffler int
11426810ad6fSSam Leffler crypto_getcaps(int hid)
11436810ad6fSSam Leffler {
1144c0341432SJohn Baldwin 	struct cryptocap *cap;
1145c0341432SJohn Baldwin 	int flags;
1146c0341432SJohn Baldwin 
1147c0341432SJohn Baldwin 	flags = 0;
1148c0341432SJohn Baldwin 	CRYPTO_DRIVER_LOCK();
1149c0341432SJohn Baldwin 	cap = crypto_checkdriver(hid);
1150c0341432SJohn Baldwin 	if (cap != NULL)
1151c0341432SJohn Baldwin 		flags = cap->cc_flags;
1152c0341432SJohn Baldwin 	CRYPTO_DRIVER_UNLOCK();
1153c0341432SJohn Baldwin 	return (flags);
1154091d81d1SSam Leffler }
1155091d81d1SSam Leffler 
1156091d81d1SSam Leffler /*
1157091d81d1SSam Leffler  * Unregister all algorithms associated with a crypto driver.
1158091d81d1SSam Leffler  * If there are pending sessions using it, leave enough information
1159091d81d1SSam Leffler  * around so that subsequent calls using those sessions will
1160091d81d1SSam Leffler  * correctly detect the driver has been unregistered and reroute
1161091d81d1SSam Leffler  * requests.
1162091d81d1SSam Leffler  */
1163091d81d1SSam Leffler int
1164d3d79e96SJohn Baldwin crypto_unregister_all(uint32_t driverid)
1165091d81d1SSam Leffler {
1166091d81d1SSam Leffler 	struct cryptocap *cap;
1167091d81d1SSam Leffler 
1168091d81d1SSam Leffler 	CRYPTO_DRIVER_LOCK();
1169091d81d1SSam Leffler 	cap = crypto_checkdriver(driverid);
1170c0341432SJohn Baldwin 	if (cap == NULL) {
1171091d81d1SSam Leffler 		CRYPTO_DRIVER_UNLOCK();
1172c0341432SJohn Baldwin 		return (EINVAL);
1173c0341432SJohn Baldwin 	}
11746810ad6fSSam Leffler 
1175c0341432SJohn Baldwin 	cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
1176c0341432SJohn Baldwin 	crypto_drivers[driverid] = NULL;
1177c0341432SJohn Baldwin 
1178c0341432SJohn Baldwin 	/*
1179c0341432SJohn Baldwin 	 * XXX: This doesn't do anything to kick sessions that
1180c0341432SJohn Baldwin 	 * have no pending operations.
1181c0341432SJohn Baldwin 	 */
118276681661SJohn Baldwin 	while (cap->cc_sessions != 0)
1183c0341432SJohn Baldwin 		mtx_sleep(cap, &crypto_drivers_mtx, 0, "cryunreg", 0);
1184c0341432SJohn Baldwin 	CRYPTO_DRIVER_UNLOCK();
1185c0341432SJohn Baldwin 	cap_rele(cap);
1186c0341432SJohn Baldwin 
1187c0341432SJohn Baldwin 	return (0);
1188091d81d1SSam Leffler }
1189091d81d1SSam Leffler 
1190091d81d1SSam Leffler /*
1191091d81d1SSam Leffler  * Clear blockage on a driver.  The what parameter indicates whether
1192091d81d1SSam Leffler  * the driver is now ready for cryptop's and/or cryptokop's.
1193091d81d1SSam Leffler  */
1194091d81d1SSam Leffler int
1195d3d79e96SJohn Baldwin crypto_unblock(uint32_t driverid, int what)
1196091d81d1SSam Leffler {
1197091d81d1SSam Leffler 	struct cryptocap *cap;
11983a865c82SPawel Jakub Dawidek 	int err;
1199091d81d1SSam Leffler 
1200091d81d1SSam Leffler 	CRYPTO_Q_LOCK();
1201091d81d1SSam Leffler 	cap = crypto_checkdriver(driverid);
1202091d81d1SSam Leffler 	if (cap != NULL) {
12033a865c82SPawel Jakub Dawidek 		if (what & CRYPTO_SYMQ)
1204091d81d1SSam Leffler 			cap->cc_qblocked = 0;
12053a865c82SPawel Jakub Dawidek 		if (crp_sleep)
12061a91ccccSSam Leffler 			wakeup_one(&crp_q);
1207091d81d1SSam Leffler 		err = 0;
1208091d81d1SSam Leffler 	} else
1209091d81d1SSam Leffler 		err = EINVAL;
1210091d81d1SSam Leffler 	CRYPTO_Q_UNLOCK();
1211091d81d1SSam Leffler 
1212091d81d1SSam Leffler 	return err;
1213091d81d1SSam Leffler }
1214091d81d1SSam Leffler 
12159c0e3d3aSJohn Baldwin size_t
12169c0e3d3aSJohn Baldwin crypto_buffer_len(struct crypto_buffer *cb)
12179c0e3d3aSJohn Baldwin {
12189c0e3d3aSJohn Baldwin 	switch (cb->cb_type) {
12199c0e3d3aSJohn Baldwin 	case CRYPTO_BUF_CONTIG:
12209c0e3d3aSJohn Baldwin 		return (cb->cb_buf_len);
12219c0e3d3aSJohn Baldwin 	case CRYPTO_BUF_MBUF:
12229c0e3d3aSJohn Baldwin 		if (cb->cb_mbuf->m_flags & M_PKTHDR)
12239c0e3d3aSJohn Baldwin 			return (cb->cb_mbuf->m_pkthdr.len);
12249c0e3d3aSJohn Baldwin 		return (m_length(cb->cb_mbuf, NULL));
1225883a0196SJohn Baldwin 	case CRYPTO_BUF_SINGLE_MBUF:
1226883a0196SJohn Baldwin 		return (cb->cb_mbuf->m_len);
1227e6f6d0c9SAlan Somers 	case CRYPTO_BUF_VMPAGE:
1228e6f6d0c9SAlan Somers 		return (cb->cb_vm_page_len);
12299c0e3d3aSJohn Baldwin 	case CRYPTO_BUF_UIO:
12309c0e3d3aSJohn Baldwin 		return (cb->cb_uio->uio_resid);
12319c0e3d3aSJohn Baldwin 	default:
12329c0e3d3aSJohn Baldwin 		return (0);
12339c0e3d3aSJohn Baldwin 	}
12349c0e3d3aSJohn Baldwin }
12359c0e3d3aSJohn Baldwin 
1236c0341432SJohn Baldwin #ifdef INVARIANTS
1237c0341432SJohn Baldwin /* Various sanity checks on crypto requests. */
1238c0341432SJohn Baldwin static void
12399c0e3d3aSJohn Baldwin cb_sanity(struct crypto_buffer *cb, const char *name)
12409c0e3d3aSJohn Baldwin {
12419c0e3d3aSJohn Baldwin 	KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST,
12429c0e3d3aSJohn Baldwin 	    ("incoming crp with invalid %s buffer type", name));
1243e6f6d0c9SAlan Somers 	switch (cb->cb_type) {
1244e6f6d0c9SAlan Somers 	case CRYPTO_BUF_CONTIG:
12459c0e3d3aSJohn Baldwin 		KASSERT(cb->cb_buf_len >= 0,
12469c0e3d3aSJohn Baldwin 		    ("incoming crp with -ve %s buffer length", name));
1247e6f6d0c9SAlan Somers 		break;
1248e6f6d0c9SAlan Somers 	case CRYPTO_BUF_VMPAGE:
1249e6f6d0c9SAlan Somers 		KASSERT(CRYPTO_HAS_VMPAGE,
1250e6f6d0c9SAlan Somers 		    ("incoming crp uses dmap on supported arch"));
1251e6f6d0c9SAlan Somers 		KASSERT(cb->cb_vm_page_len >= 0,
1252e6f6d0c9SAlan Somers 		    ("incoming crp with -ve %s buffer length", name));
1253e6f6d0c9SAlan Somers 		KASSERT(cb->cb_vm_page_offset >= 0,
1254e6f6d0c9SAlan Somers 		    ("incoming crp with -ve %s buffer offset", name));
1255e6f6d0c9SAlan Somers 		KASSERT(cb->cb_vm_page_offset < PAGE_SIZE,
1256e6f6d0c9SAlan Somers 		    ("incoming crp with %s buffer offset greater than page size"
1257e6f6d0c9SAlan Somers 		     , name));
1258e6f6d0c9SAlan Somers 		break;
1259e6f6d0c9SAlan Somers 	default:
1260e6f6d0c9SAlan Somers 		break;
1261e6f6d0c9SAlan Somers 	}
12629c0e3d3aSJohn Baldwin }
12639c0e3d3aSJohn Baldwin 
12649c0e3d3aSJohn Baldwin static void
1265c0341432SJohn Baldwin crp_sanity(struct cryptop *crp)
1266c0341432SJohn Baldwin {
1267c0341432SJohn Baldwin 	struct crypto_session_params *csp;
12689c0e3d3aSJohn Baldwin 	struct crypto_buffer *out;
12699c0e3d3aSJohn Baldwin 	size_t ilen, len, olen;
1270c0341432SJohn Baldwin 
1271c0341432SJohn Baldwin 	KASSERT(crp->crp_session != NULL, ("incoming crp without a session"));
12729c0e3d3aSJohn Baldwin 	KASSERT(crp->crp_obuf.cb_type >= CRYPTO_BUF_NONE &&
12739c0e3d3aSJohn Baldwin 	    crp->crp_obuf.cb_type <= CRYPTO_BUF_LAST,
12749c0e3d3aSJohn Baldwin 	    ("incoming crp with invalid output buffer type"));
1275c0341432SJohn Baldwin 	KASSERT(crp->crp_etype == 0, ("incoming crp with error"));
1276c0341432SJohn Baldwin 	KASSERT(!(crp->crp_flags & CRYPTO_F_DONE),
1277c0341432SJohn Baldwin 	    ("incoming crp already done"));
1278c0341432SJohn Baldwin 
1279c0341432SJohn Baldwin 	csp = &crp->crp_session->csp;
12809c0e3d3aSJohn Baldwin 	cb_sanity(&crp->crp_buf, "input");
12819c0e3d3aSJohn Baldwin 	ilen = crypto_buffer_len(&crp->crp_buf);
12829c0e3d3aSJohn Baldwin 	olen = ilen;
12839c0e3d3aSJohn Baldwin 	out = NULL;
12849c0e3d3aSJohn Baldwin 	if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT) {
12859c0e3d3aSJohn Baldwin 		if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE) {
12869c0e3d3aSJohn Baldwin 			cb_sanity(&crp->crp_obuf, "output");
12879c0e3d3aSJohn Baldwin 			out = &crp->crp_obuf;
12889c0e3d3aSJohn Baldwin 			olen = crypto_buffer_len(out);
12899c0e3d3aSJohn Baldwin 		}
12909c0e3d3aSJohn Baldwin 	} else
12919c0e3d3aSJohn Baldwin 		KASSERT(crp->crp_obuf.cb_type == CRYPTO_BUF_NONE,
12929c0e3d3aSJohn Baldwin 		    ("incoming crp with separate output buffer "
12939c0e3d3aSJohn Baldwin 		    "but no session support"));
12949c0e3d3aSJohn Baldwin 
1295c0341432SJohn Baldwin 	switch (csp->csp_mode) {
1296c0341432SJohn Baldwin 	case CSP_MODE_COMPRESS:
1297c0341432SJohn Baldwin 		KASSERT(crp->crp_op == CRYPTO_OP_COMPRESS ||
1298c0341432SJohn Baldwin 		    crp->crp_op == CRYPTO_OP_DECOMPRESS,
1299c0341432SJohn Baldwin 		    ("invalid compression op %x", crp->crp_op));
1300c0341432SJohn Baldwin 		break;
1301c0341432SJohn Baldwin 	case CSP_MODE_CIPHER:
1302c0341432SJohn Baldwin 		KASSERT(crp->crp_op == CRYPTO_OP_ENCRYPT ||
1303c0341432SJohn Baldwin 		    crp->crp_op == CRYPTO_OP_DECRYPT,
1304c0341432SJohn Baldwin 		    ("invalid cipher op %x", crp->crp_op));
1305c0341432SJohn Baldwin 		break;
1306c0341432SJohn Baldwin 	case CSP_MODE_DIGEST:
1307c0341432SJohn Baldwin 		KASSERT(crp->crp_op == CRYPTO_OP_COMPUTE_DIGEST ||
1308c0341432SJohn Baldwin 		    crp->crp_op == CRYPTO_OP_VERIFY_DIGEST,
1309c0341432SJohn Baldwin 		    ("invalid digest op %x", crp->crp_op));
1310c0341432SJohn Baldwin 		break;
1311c0341432SJohn Baldwin 	case CSP_MODE_AEAD:
1312c0341432SJohn Baldwin 		KASSERT(crp->crp_op ==
1313c0341432SJohn Baldwin 		    (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
1314c0341432SJohn Baldwin 		    crp->crp_op ==
1315c0341432SJohn Baldwin 		    (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
1316c0341432SJohn Baldwin 		    ("invalid AEAD op %x", crp->crp_op));
1317c0341432SJohn Baldwin 		KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE,
1318fc8fc743SJohn Baldwin 		    ("AEAD without a separate IV"));
1319c0341432SJohn Baldwin 		break;
1320c0341432SJohn Baldwin 	case CSP_MODE_ETA:
1321c0341432SJohn Baldwin 		KASSERT(crp->crp_op ==
1322c0341432SJohn Baldwin 		    (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
1323c0341432SJohn Baldwin 		    crp->crp_op ==
1324c0341432SJohn Baldwin 		    (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
1325c0341432SJohn Baldwin 		    ("invalid ETA op %x", crp->crp_op));
1326c0341432SJohn Baldwin 		break;
1327c0341432SJohn Baldwin 	}
1328c0341432SJohn Baldwin 	if (csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
13299b774dc0SJohn Baldwin 		if (crp->crp_aad == NULL) {
1330c0341432SJohn Baldwin 			KASSERT(crp->crp_aad_start == 0 ||
13319c0e3d3aSJohn Baldwin 			    crp->crp_aad_start < ilen,
1332c0341432SJohn Baldwin 			    ("invalid AAD start"));
13339b774dc0SJohn Baldwin 			KASSERT(crp->crp_aad_length != 0 ||
13349b774dc0SJohn Baldwin 			    crp->crp_aad_start == 0,
1335c0341432SJohn Baldwin 			    ("AAD with zero length and non-zero start"));
1336c0341432SJohn Baldwin 			KASSERT(crp->crp_aad_length == 0 ||
13379c0e3d3aSJohn Baldwin 			    crp->crp_aad_start + crp->crp_aad_length <= ilen,
1338c0341432SJohn Baldwin 			    ("AAD outside input length"));
1339c0341432SJohn Baldwin 		} else {
13409b774dc0SJohn Baldwin 			KASSERT(csp->csp_flags & CSP_F_SEPARATE_AAD,
13419b774dc0SJohn Baldwin 			    ("session doesn't support separate AAD buffer"));
13429b774dc0SJohn Baldwin 			KASSERT(crp->crp_aad_start == 0,
13439b774dc0SJohn Baldwin 			    ("separate AAD buffer with non-zero AAD start"));
13449b774dc0SJohn Baldwin 			KASSERT(crp->crp_aad_length != 0,
13459b774dc0SJohn Baldwin 			    ("separate AAD buffer with zero length"));
13469b774dc0SJohn Baldwin 		}
13479b774dc0SJohn Baldwin 	} else {
13489b774dc0SJohn Baldwin 		KASSERT(crp->crp_aad == NULL && crp->crp_aad_start == 0 &&
13499b774dc0SJohn Baldwin 		    crp->crp_aad_length == 0,
1350c0341432SJohn Baldwin 		    ("AAD region in request not supporting AAD"));
1351c0341432SJohn Baldwin 	}
1352c0341432SJohn Baldwin 	if (csp->csp_ivlen == 0) {
135329fe41ddSJohn Baldwin 		KASSERT((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0,
135429fe41ddSJohn Baldwin 		    ("IV_SEPARATE set when IV isn't used"));
1355c0341432SJohn Baldwin 		KASSERT(crp->crp_iv_start == 0,
1356c0341432SJohn Baldwin 		    ("crp_iv_start set when IV isn't used"));
1357c0341432SJohn Baldwin 	} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
1358c0341432SJohn Baldwin 		KASSERT(crp->crp_iv_start == 0,
1359c0341432SJohn Baldwin 		    ("IV_SEPARATE used with non-zero IV start"));
1360c0341432SJohn Baldwin 	} else {
13619c0e3d3aSJohn Baldwin 		KASSERT(crp->crp_iv_start < ilen,
1362c0341432SJohn Baldwin 		    ("invalid IV start"));
13639c0e3d3aSJohn Baldwin 		KASSERT(crp->crp_iv_start + csp->csp_ivlen <= ilen,
13649c0e3d3aSJohn Baldwin 		    ("IV outside buffer length"));
1365c0341432SJohn Baldwin 	}
13669c0e3d3aSJohn Baldwin 	/* XXX: payload_start of 0 should always be < ilen? */
1367c0341432SJohn Baldwin 	KASSERT(crp->crp_payload_start == 0 ||
13689c0e3d3aSJohn Baldwin 	    crp->crp_payload_start < ilen,
1369c0341432SJohn Baldwin 	    ("invalid payload start"));
1370c0341432SJohn Baldwin 	KASSERT(crp->crp_payload_start + crp->crp_payload_length <=
13719c0e3d3aSJohn Baldwin 	    ilen, ("payload outside input buffer"));
13729c0e3d3aSJohn Baldwin 	if (out == NULL) {
13739c0e3d3aSJohn Baldwin 		KASSERT(crp->crp_payload_output_start == 0,
13749c0e3d3aSJohn Baldwin 		    ("payload output start non-zero without output buffer"));
13759c0e3d3aSJohn Baldwin 	} else {
1376ec498562SJohn Baldwin 		KASSERT(crp->crp_payload_output_start == 0 ||
1377ec498562SJohn Baldwin 		    crp->crp_payload_output_start < olen,
13789c0e3d3aSJohn Baldwin 		    ("invalid payload output start"));
13799c0e3d3aSJohn Baldwin 		KASSERT(crp->crp_payload_output_start +
13809c0e3d3aSJohn Baldwin 		    crp->crp_payload_length <= olen,
13819c0e3d3aSJohn Baldwin 		    ("payload outside output buffer"));
13829c0e3d3aSJohn Baldwin 	}
1383c0341432SJohn Baldwin 	if (csp->csp_mode == CSP_MODE_DIGEST ||
1384c0341432SJohn Baldwin 	    csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
13859c0e3d3aSJohn Baldwin 		if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST)
13869c0e3d3aSJohn Baldwin 			len = ilen;
13879c0e3d3aSJohn Baldwin 		else
13889c0e3d3aSJohn Baldwin 			len = olen;
1389c0341432SJohn Baldwin 		KASSERT(crp->crp_digest_start == 0 ||
13909c0e3d3aSJohn Baldwin 		    crp->crp_digest_start < len,
1391c0341432SJohn Baldwin 		    ("invalid digest start"));
1392c0341432SJohn Baldwin 		/* XXX: For the mlen == 0 case this check isn't perfect. */
13939c0e3d3aSJohn Baldwin 		KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <= len,
13949c0e3d3aSJohn Baldwin 		    ("digest outside buffer"));
1395c0341432SJohn Baldwin 	} else {
1396c0341432SJohn Baldwin 		KASSERT(crp->crp_digest_start == 0,
1397c0341432SJohn Baldwin 		    ("non-zero digest start for request without a digest"));
1398c0341432SJohn Baldwin 	}
1399c0341432SJohn Baldwin 	if (csp->csp_cipher_klen != 0)
1400c0341432SJohn Baldwin 		KASSERT(csp->csp_cipher_key != NULL ||
1401c0341432SJohn Baldwin 		    crp->crp_cipher_key != NULL,
1402c0341432SJohn Baldwin 		    ("cipher request without a key"));
1403c0341432SJohn Baldwin 	if (csp->csp_auth_klen != 0)
1404c0341432SJohn Baldwin 		KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL,
1405c0341432SJohn Baldwin 		    ("auth request without a key"));
1406c0341432SJohn Baldwin 	KASSERT(crp->crp_callback != NULL, ("incoming crp without callback"));
1407c0341432SJohn Baldwin }
1408c0341432SJohn Baldwin #endif
1409c0341432SJohn Baldwin 
141068f6800cSMark Johnston static int
141168f6800cSMark Johnston crypto_dispatch_one(struct cryptop *crp, int hint)
1412091d81d1SSam Leffler {
14134acae0acSPawel Jakub Dawidek 	struct cryptocap *cap;
14144acae0acSPawel Jakub Dawidek 	int result;
1415091d81d1SSam Leffler 
1416c0341432SJohn Baldwin #ifdef INVARIANTS
1417c0341432SJohn Baldwin 	crp_sanity(crp);
1418c0341432SJohn Baldwin #endif
14197290cb47SMark Johnston 	CRYPTOSTAT_INC(cs_ops);
14207d1853eeSSam Leffler 
142198d788c8SMark Johnston 	crp->crp_retw_id = crp->crp_session->id % crypto_workers_num;
1422de2b2c90SFabien Thomas 
142368f6800cSMark Johnston 	/*
142468f6800cSMark Johnston 	 * Caller marked the request to be processed immediately; dispatch it
142568f6800cSMark Johnston 	 * directly to the driver unless the driver is currently blocked, in
142668f6800cSMark Johnston 	 * which case it is queued for deferred dispatch.
142768f6800cSMark Johnston 	 */
142868f6800cSMark Johnston 	cap = crp->crp_session->cap;
142968f6800cSMark Johnston 	if (!atomic_load_int(&cap->cc_qblocked)) {
143068f6800cSMark Johnston 		result = crypto_invoke(cap, crp, hint);
143168f6800cSMark Johnston 		if (result != ERESTART)
143268f6800cSMark Johnston 			return (result);
143368f6800cSMark Johnston 
143468f6800cSMark Johnston 		/*
143568f6800cSMark Johnston 		 * The driver ran out of resources, put the request on the
143668f6800cSMark Johnston 		 * queue.
143768f6800cSMark Johnston 		 */
143868f6800cSMark Johnston 	}
143968f6800cSMark Johnston 	crypto_batch_enqueue(crp);
144068f6800cSMark Johnston 	return (0);
144168f6800cSMark Johnston }
144268f6800cSMark Johnston 
144368f6800cSMark Johnston int
144468f6800cSMark Johnston crypto_dispatch(struct cryptop *crp)
144568f6800cSMark Johnston {
144668f6800cSMark Johnston 	return (crypto_dispatch_one(crp, 0));
144768f6800cSMark Johnston }
144868f6800cSMark Johnston 
144968f6800cSMark Johnston int
145068f6800cSMark Johnston crypto_dispatch_async(struct cryptop *crp, int flags)
145168f6800cSMark Johnston {
145239bbca6fSFabien Thomas 	struct crypto_ret_worker *ret_worker;
145339bbca6fSFabien Thomas 
145468f6800cSMark Johnston 	if (!CRYPTO_SESS_SYNC(crp->crp_session)) {
145568f6800cSMark Johnston 		/*
145668f6800cSMark Johnston 		 * The driver issues completions asynchonously, don't bother
145768f6800cSMark Johnston 		 * deferring dispatch to a worker thread.
145868f6800cSMark Johnston 		 */
145968f6800cSMark Johnston 		return (crypto_dispatch(crp));
146068f6800cSMark Johnston 	}
146139bbca6fSFabien Thomas 
146268f6800cSMark Johnston #ifdef INVARIANTS
146368f6800cSMark Johnston 	crp_sanity(crp);
146468f6800cSMark Johnston #endif
146568f6800cSMark Johnston 	CRYPTOSTAT_INC(cs_ops);
146668f6800cSMark Johnston 
146768f6800cSMark Johnston 	crp->crp_retw_id = crp->crp_session->id % crypto_workers_num;
146868f6800cSMark Johnston 	if ((flags & CRYPTO_ASYNC_ORDERED) != 0) {
146968f6800cSMark Johnston 		crp->crp_flags |= CRYPTO_F_ASYNC_ORDERED;
147068f6800cSMark Johnston 		ret_worker = CRYPTO_RETW(crp->crp_retw_id);
147139bbca6fSFabien Thomas 		CRYPTO_RETW_LOCK(ret_worker);
147239bbca6fSFabien Thomas 		crp->crp_seq = ret_worker->reorder_ops++;
147339bbca6fSFabien Thomas 		CRYPTO_RETW_UNLOCK(ret_worker);
147439bbca6fSFabien Thomas 	}
147539bbca6fSFabien Thomas 	TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp);
147639bbca6fSFabien Thomas 	taskqueue_enqueue(crypto_tq, &crp->crp_task);
147739bbca6fSFabien Thomas 	return (0);
147839bbca6fSFabien Thomas }
14794acae0acSPawel Jakub Dawidek 
148068f6800cSMark Johnston void
148168f6800cSMark Johnston crypto_dispatch_batch(struct cryptopq *crpq, int flags)
148268f6800cSMark Johnston {
148368f6800cSMark Johnston 	struct cryptop *crp;
148468f6800cSMark Johnston 	int hint;
148568f6800cSMark Johnston 
148668f6800cSMark Johnston 	while ((crp = TAILQ_FIRST(crpq)) != NULL) {
148768f6800cSMark Johnston 		hint = TAILQ_NEXT(crp, crp_next) != NULL ? CRYPTO_HINT_MORE : 0;
148868f6800cSMark Johnston 		TAILQ_REMOVE(crpq, crp, crp_next);
148968f6800cSMark Johnston 		if (crypto_dispatch_one(crp, hint) != 0)
149039bbca6fSFabien Thomas 			crypto_batch_enqueue(crp);
149168f6800cSMark Johnston 	}
149239bbca6fSFabien Thomas }
149339bbca6fSFabien Thomas 
149468f6800cSMark Johnston static void
149539bbca6fSFabien Thomas crypto_batch_enqueue(struct cryptop *crp)
149639bbca6fSFabien Thomas {
149739bbca6fSFabien Thomas 
14984acae0acSPawel Jakub Dawidek 	CRYPTO_Q_LOCK();
14994acae0acSPawel Jakub Dawidek 	TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
15003a865c82SPawel Jakub Dawidek 	if (crp_sleep)
15013a865c82SPawel Jakub Dawidek 		wakeup_one(&crp_q);
15023569ae7fSSam Leffler 	CRYPTO_Q_UNLOCK();
1503091d81d1SSam Leffler }
1504091d81d1SSam Leffler 
150539bbca6fSFabien Thomas static void
150639bbca6fSFabien Thomas crypto_task_invoke(void *ctx, int pending)
150739bbca6fSFabien Thomas {
150839bbca6fSFabien Thomas 	struct cryptocap *cap;
150939bbca6fSFabien Thomas 	struct cryptop *crp;
1510c0341432SJohn Baldwin 	int result;
151139bbca6fSFabien Thomas 
151239bbca6fSFabien Thomas 	crp = (struct cryptop *)ctx;
1513c0341432SJohn Baldwin 	cap = crp->crp_session->cap;
151439bbca6fSFabien Thomas 	result = crypto_invoke(cap, crp, 0);
151539bbca6fSFabien Thomas 	if (result == ERESTART)
151639bbca6fSFabien Thomas 		crypto_batch_enqueue(crp);
151739bbca6fSFabien Thomas }
151839bbca6fSFabien Thomas 
1519091d81d1SSam Leffler /*
1520091d81d1SSam Leffler  * Dispatch a crypto request to the appropriate crypto devices.
1521091d81d1SSam Leffler  */
1522091d81d1SSam Leffler static int
15234acae0acSPawel Jakub Dawidek crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint)
1524091d81d1SSam Leffler {
15254acae0acSPawel Jakub Dawidek 
15264acae0acSPawel Jakub Dawidek 	KASSERT(crp != NULL, ("%s: crp == NULL", __func__));
15274acae0acSPawel Jakub Dawidek 	KASSERT(crp->crp_callback != NULL,
15284acae0acSPawel Jakub Dawidek 	    ("%s: crp->crp_callback == NULL", __func__));
1529c0341432SJohn Baldwin 	KASSERT(crp->crp_session != NULL,
1530c0341432SJohn Baldwin 	    ("%s: crp->crp_session == NULL", __func__));
1531091d81d1SSam Leffler 
15324acae0acSPawel Jakub Dawidek 	if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1533c0341432SJohn Baldwin 		struct crypto_session_params csp;
15341b0909d5SConrad Meyer 		crypto_session_t nses;
1535091d81d1SSam Leffler 
1536091d81d1SSam Leffler 		/*
1537091d81d1SSam Leffler 		 * Driver has unregistered; migrate the session and return
1538091d81d1SSam Leffler 		 * an error to the caller so they'll resubmit the op.
15394acae0acSPawel Jakub Dawidek 		 *
15404acae0acSPawel Jakub Dawidek 		 * XXX: What if there are more already queued requests for this
15414acae0acSPawel Jakub Dawidek 		 *      session?
1542c0341432SJohn Baldwin 		 *
1543c0341432SJohn Baldwin 		 * XXX: Real solution is to make sessions refcounted
1544c0341432SJohn Baldwin 		 * and force callers to hold a reference when
1545c0341432SJohn Baldwin 		 * assigning to crp_session.  Could maybe change
1546c0341432SJohn Baldwin 		 * crypto_getreq to accept a session pointer to make
1547c0341432SJohn Baldwin 		 * that work.  Alternatively, we could abandon the
1548c0341432SJohn Baldwin 		 * notion of rewriting crp_session in requests forcing
1549c0341432SJohn Baldwin 		 * the caller to deal with allocating a new session.
1550c0341432SJohn Baldwin 		 * Perhaps provide a method to allow a crp's session to
1551c0341432SJohn Baldwin 		 * be swapped that callers could use.
1552091d81d1SSam Leffler 		 */
1553c0341432SJohn Baldwin 		csp = crp->crp_session->csp;
15541b0909d5SConrad Meyer 		crypto_freesession(crp->crp_session);
15554acae0acSPawel Jakub Dawidek 
1556c0341432SJohn Baldwin 		/*
1557c0341432SJohn Baldwin 		 * XXX: Key pointers may no longer be valid.  If we
1558c0341432SJohn Baldwin 		 * really want to support this we need to define the
1559c0341432SJohn Baldwin 		 * KPI such that 'csp' is required to be valid for the
1560c0341432SJohn Baldwin 		 * duration of a session by the caller perhaps.
1561c0341432SJohn Baldwin 		 *
1562c0341432SJohn Baldwin 		 * XXX: If the keys have been changed this will reuse
1563c0341432SJohn Baldwin 		 * the old keys.  This probably suggests making
1564c0341432SJohn Baldwin 		 * rekeying more explicit and updating the key
1565c0341432SJohn Baldwin 		 * pointers in 'csp' when the keys change.
1566c0341432SJohn Baldwin 		 */
1567c0341432SJohn Baldwin 		if (crypto_newsession(&nses, &csp,
15686810ad6fSSam Leffler 		    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0)
15691b0909d5SConrad Meyer 			crp->crp_session = nses;
1570091d81d1SSam Leffler 
1571091d81d1SSam Leffler 		crp->crp_etype = EAGAIN;
15721a91ccccSSam Leffler 		crypto_done(crp);
1573091d81d1SSam Leffler 		return 0;
1574091d81d1SSam Leffler 	} else {
1575091d81d1SSam Leffler 		/*
1576091d81d1SSam Leffler 		 * Invoke the driver to process the request.
1577091d81d1SSam Leffler 		 */
15786810ad6fSSam Leffler 		return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint);
1579091d81d1SSam Leffler 	}
1580091d81d1SSam Leffler }
1581091d81d1SSam Leffler 
1582091d81d1SSam Leffler void
1583946b8f6fSJohn Baldwin crypto_destroyreq(struct cryptop *crp)
1584091d81d1SSam Leffler {
15850d5c337bSPawel Jakub Dawidek #ifdef DIAGNOSTIC
15860d5c337bSPawel Jakub Dawidek 	{
15870d5c337bSPawel Jakub Dawidek 		struct cryptop *crp2;
158839bbca6fSFabien Thomas 		struct crypto_ret_worker *ret_worker;
15890d5c337bSPawel Jakub Dawidek 
1590*70439285SMateusz Guzik 		if (!crypto_destroyreq_check)
1591*70439285SMateusz Guzik 			return;
1592*70439285SMateusz Guzik 
15930d5c337bSPawel Jakub Dawidek 		CRYPTO_Q_LOCK();
15940d5c337bSPawel Jakub Dawidek 		TAILQ_FOREACH(crp2, &crp_q, crp_next) {
15950d5c337bSPawel Jakub Dawidek 			KASSERT(crp2 != crp,
15960d5c337bSPawel Jakub Dawidek 			    ("Freeing cryptop from the crypto queue (%p).",
15970d5c337bSPawel Jakub Dawidek 			    crp));
15980d5c337bSPawel Jakub Dawidek 		}
15990d5c337bSPawel Jakub Dawidek 		CRYPTO_Q_UNLOCK();
160039bbca6fSFabien Thomas 
160139bbca6fSFabien Thomas 		FOREACH_CRYPTO_RETW(ret_worker) {
160239bbca6fSFabien Thomas 			CRYPTO_RETW_LOCK(ret_worker);
160339bbca6fSFabien Thomas 			TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) {
16040d5c337bSPawel Jakub Dawidek 				KASSERT(crp2 != crp,
16050d5c337bSPawel Jakub Dawidek 				    ("Freeing cryptop from the return queue (%p).",
16060d5c337bSPawel Jakub Dawidek 				    crp));
16070d5c337bSPawel Jakub Dawidek 			}
160839bbca6fSFabien Thomas 			CRYPTO_RETW_UNLOCK(ret_worker);
160939bbca6fSFabien Thomas 		}
16100d5c337bSPawel Jakub Dawidek 	}
16110d5c337bSPawel Jakub Dawidek #endif
1612946b8f6fSJohn Baldwin }
16130d5c337bSPawel Jakub Dawidek 
1614946b8f6fSJohn Baldwin void
1615946b8f6fSJohn Baldwin crypto_freereq(struct cryptop *crp)
1616946b8f6fSJohn Baldwin {
1617946b8f6fSJohn Baldwin 	if (crp == NULL)
1618946b8f6fSJohn Baldwin 		return;
1619946b8f6fSJohn Baldwin 
1620946b8f6fSJohn Baldwin 	crypto_destroyreq(crp);
1621091d81d1SSam Leffler 	uma_zfree(cryptop_zone, crp);
1622091d81d1SSam Leffler }
1623091d81d1SSam Leffler 
1624946b8f6fSJohn Baldwin static void
1625946b8f6fSJohn Baldwin _crypto_initreq(struct cryptop *crp, crypto_session_t cses)
1626946b8f6fSJohn Baldwin {
1627946b8f6fSJohn Baldwin 	crp->crp_session = cses;
1628946b8f6fSJohn Baldwin }
1629946b8f6fSJohn Baldwin 
1630946b8f6fSJohn Baldwin void
1631946b8f6fSJohn Baldwin crypto_initreq(struct cryptop *crp, crypto_session_t cses)
1632946b8f6fSJohn Baldwin {
1633946b8f6fSJohn Baldwin 	memset(crp, 0, sizeof(*crp));
1634946b8f6fSJohn Baldwin 	_crypto_initreq(crp, cses);
1635946b8f6fSJohn Baldwin }
1636946b8f6fSJohn Baldwin 
1637091d81d1SSam Leffler struct cryptop *
1638c0341432SJohn Baldwin crypto_getreq(crypto_session_t cses, int how)
1639091d81d1SSam Leffler {
1640091d81d1SSam Leffler 	struct cryptop *crp;
1641091d81d1SSam Leffler 
1642c0341432SJohn Baldwin 	MPASS(how == M_WAITOK || how == M_NOWAIT);
1643c0341432SJohn Baldwin 	crp = uma_zalloc(cryptop_zone, how | M_ZERO);
1644946b8f6fSJohn Baldwin 	if (crp != NULL)
1645946b8f6fSJohn Baldwin 		_crypto_initreq(crp, cses);
1646c0341432SJohn Baldwin 	return (crp);
1647091d81d1SSam Leffler }
1648091d81d1SSam Leffler 
1649091d81d1SSam Leffler /*
165074d3f1b6SJohn Baldwin  * Clone a crypto request, but associate it with the specified session
165174d3f1b6SJohn Baldwin  * rather than inheriting the session from the original request.  The
165274d3f1b6SJohn Baldwin  * fields describing the request buffers are copied, but not the
165374d3f1b6SJohn Baldwin  * opaque field or callback function.
165474d3f1b6SJohn Baldwin  */
165574d3f1b6SJohn Baldwin struct cryptop *
165674d3f1b6SJohn Baldwin crypto_clonereq(struct cryptop *crp, crypto_session_t cses, int how)
165774d3f1b6SJohn Baldwin {
165874d3f1b6SJohn Baldwin 	struct cryptop *new;
165974d3f1b6SJohn Baldwin 
166074d3f1b6SJohn Baldwin 	MPASS((crp->crp_flags & CRYPTO_F_DONE) == 0);
166174d3f1b6SJohn Baldwin 	new = crypto_getreq(cses, how);
166274d3f1b6SJohn Baldwin 	if (new == NULL)
166374d3f1b6SJohn Baldwin 		return (NULL);
166474d3f1b6SJohn Baldwin 
166574d3f1b6SJohn Baldwin 	memcpy(&new->crp_startcopy, &crp->crp_startcopy,
166674d3f1b6SJohn Baldwin 	    __rangeof(struct cryptop, crp_startcopy, crp_endcopy));
166774d3f1b6SJohn Baldwin 	return (new);
166874d3f1b6SJohn Baldwin }
166974d3f1b6SJohn Baldwin 
167074d3f1b6SJohn Baldwin /*
1671091d81d1SSam Leffler  * Invoke the callback on behalf of the driver.
1672091d81d1SSam Leffler  */
1673091d81d1SSam Leffler void
1674091d81d1SSam Leffler crypto_done(struct cryptop *crp)
1675091d81d1SSam Leffler {
16763569ae7fSSam Leffler 	KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
16773569ae7fSSam Leffler 		("crypto_done: op already done, flags 0x%x", crp->crp_flags));
16783569ae7fSSam Leffler 	crp->crp_flags |= CRYPTO_F_DONE;
16797d1853eeSSam Leffler 	if (crp->crp_etype != 0)
16807290cb47SMark Johnston 		CRYPTOSTAT_INC(cs_errs);
1681a5c053f5SMark Johnston 
1682d8409aafSSam Leffler 	/*
1683d8409aafSSam Leffler 	 * CBIMM means unconditionally do the callback immediately;
1684d8409aafSSam Leffler 	 * CBIFSYNC means do the callback immediately only if the
1685d8409aafSSam Leffler 	 * operation was done synchronously.  Both are used to avoid
1686d8409aafSSam Leffler 	 * doing extraneous context switches; the latter is mostly
1687d8409aafSSam Leffler 	 * used with the software crypto driver.
1688d8409aafSSam Leffler 	 */
168968f6800cSMark Johnston 	if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) == 0 &&
169068f6800cSMark Johnston 	    ((crp->crp_flags & CRYPTO_F_CBIMM) != 0 ||
169168f6800cSMark Johnston 	    ((crp->crp_flags & CRYPTO_F_CBIFSYNC) != 0 &&
169268f6800cSMark Johnston 	    CRYPTO_SESS_SYNC(crp->crp_session)))) {
1693eb73a605SSam Leffler 		/*
1694eb73a605SSam Leffler 		 * Do the callback directly.  This is ok when the
1695eb73a605SSam Leffler 		 * callback routine does very little (e.g. the
1696eb73a605SSam Leffler 		 * /dev/crypto callback method just does a wakeup).
1697eb73a605SSam Leffler 		 */
1698eb73a605SSam Leffler 		crp->crp_callback(crp);
1699eb73a605SSam Leffler 	} else {
170039bbca6fSFabien Thomas 		struct crypto_ret_worker *ret_worker;
170139bbca6fSFabien Thomas 		bool wake;
170239bbca6fSFabien Thomas 
170339bbca6fSFabien Thomas 		ret_worker = CRYPTO_RETW(crp->crp_retw_id);
170439bbca6fSFabien Thomas 
1705eb73a605SSam Leffler 		/*
1706eb73a605SSam Leffler 		 * Normal case; queue the callback for the thread.
1707eb73a605SSam Leffler 		 */
170839bbca6fSFabien Thomas 		CRYPTO_RETW_LOCK(ret_worker);
170968f6800cSMark Johnston 		if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) != 0) {
171039bbca6fSFabien Thomas 			struct cryptop *tmp;
171139bbca6fSFabien Thomas 
171268f6800cSMark Johnston 			TAILQ_FOREACH_REVERSE(tmp,
171368f6800cSMark Johnston 			    &ret_worker->crp_ordered_ret_q, cryptop_q,
171468f6800cSMark Johnston 			    crp_next) {
171539bbca6fSFabien Thomas 				if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) {
171668f6800cSMark Johnston 					TAILQ_INSERT_AFTER(
171768f6800cSMark Johnston 					    &ret_worker->crp_ordered_ret_q, tmp,
171868f6800cSMark Johnston 					    crp, crp_next);
171939bbca6fSFabien Thomas 					break;
172039bbca6fSFabien Thomas 				}
172139bbca6fSFabien Thomas 			}
172239bbca6fSFabien Thomas 			if (tmp == NULL) {
172368f6800cSMark Johnston 				TAILQ_INSERT_HEAD(
172468f6800cSMark Johnston 				    &ret_worker->crp_ordered_ret_q, crp,
172568f6800cSMark Johnston 				    crp_next);
172639bbca6fSFabien Thomas 			}
172739bbca6fSFabien Thomas 
172868f6800cSMark Johnston 			wake = crp->crp_seq == ret_worker->reorder_cur_seq;
172968f6800cSMark Johnston 		} else {
173068f6800cSMark Johnston 			wake = TAILQ_EMPTY(&ret_worker->crp_ret_q);
173168f6800cSMark Johnston 			TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp,
173268f6800cSMark Johnston 			    crp_next);
173339bbca6fSFabien Thomas 		}
173439bbca6fSFabien Thomas 
173539bbca6fSFabien Thomas 		if (wake)
173639bbca6fSFabien Thomas 			wakeup_one(&ret_worker->crp_ret_q);	/* shared wait channel */
173739bbca6fSFabien Thomas 		CRYPTO_RETW_UNLOCK(ret_worker);
1738091d81d1SSam Leffler 	}
1739eb73a605SSam Leffler }
1740091d81d1SSam Leffler 
1741091d81d1SSam Leffler /*
174251e45326SSam Leffler  * Terminate a thread at module unload.  The process that
174351e45326SSam Leffler  * initiated this is waiting for us to signal that we're gone;
174451e45326SSam Leffler  * wake it up and exit.  We use the driver table lock to insure
174551e45326SSam Leffler  * we don't do the wakeup before they're waiting.  There is no
174651e45326SSam Leffler  * race here because the waiter sleeps on the proc lock for the
174751e45326SSam Leffler  * thread so it gets notified at the right time because of an
174851e45326SSam Leffler  * extra wakeup that's done in exit1().
174951e45326SSam Leffler  */
1750091d81d1SSam Leffler static void
175151e45326SSam Leffler crypto_finis(void *chan)
1752091d81d1SSam Leffler {
175351e45326SSam Leffler 	CRYPTO_DRIVER_LOCK();
175451e45326SSam Leffler 	wakeup_one(chan);
175551e45326SSam Leffler 	CRYPTO_DRIVER_UNLOCK();
175671785781SJohn Baldwin 	kthread_exit();
1757091d81d1SSam Leffler }
1758091d81d1SSam Leffler 
1759091d81d1SSam Leffler /*
17601a91ccccSSam Leffler  * Crypto thread, dispatches crypto requests.
1761091d81d1SSam Leffler  */
1762091d81d1SSam Leffler static void
176371785781SJohn Baldwin crypto_dispatch_thread(void *arg __unused)
1764091d81d1SSam Leffler {
17651a91ccccSSam Leffler 	struct cryptop *crp, *submit;
1766091d81d1SSam Leffler 	struct cryptocap *cap;
1767091d81d1SSam Leffler 	int result, hint;
1768091d81d1SSam Leffler 
17696ed982a2SAndrew Turner #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
177004c49e68SKonstantin Belousov 	fpu_kern_thread(FPU_KERN_NORMAL);
177104c49e68SKonstantin Belousov #endif
177204c49e68SKonstantin Belousov 
17731a91ccccSSam Leffler 	CRYPTO_Q_LOCK();
1774091d81d1SSam Leffler 	for (;;) {
1775091d81d1SSam Leffler 		/*
1776091d81d1SSam Leffler 		 * Find the first element in the queue that can be
1777091d81d1SSam Leffler 		 * processed and look-ahead to see if multiple ops
1778091d81d1SSam Leffler 		 * are ready for the same driver.
1779091d81d1SSam Leffler 		 */
1780091d81d1SSam Leffler 		submit = NULL;
1781091d81d1SSam Leffler 		hint = 0;
1782091d81d1SSam Leffler 		TAILQ_FOREACH(crp, &crp_q, crp_next) {
1783c0341432SJohn Baldwin 			cap = crp->crp_session->cap;
17844acae0acSPawel Jakub Dawidek 			/*
17854acae0acSPawel Jakub Dawidek 			 * Driver cannot disappeared when there is an active
17864acae0acSPawel Jakub Dawidek 			 * session.
17874acae0acSPawel Jakub Dawidek 			 */
1788c3c82036SPawel Jakub Dawidek 			KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1789c3c82036SPawel Jakub Dawidek 			    __func__, __LINE__));
1790c0341432SJohn Baldwin 			if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1791091d81d1SSam Leffler 				/* Op needs to be migrated, process it. */
1792091d81d1SSam Leffler 				if (submit == NULL)
1793091d81d1SSam Leffler 					submit = crp;
1794091d81d1SSam Leffler 				break;
1795091d81d1SSam Leffler 			}
1796091d81d1SSam Leffler 			if (!cap->cc_qblocked) {
1797091d81d1SSam Leffler 				if (submit != NULL) {
1798091d81d1SSam Leffler 					/*
1799091d81d1SSam Leffler 					 * We stop on finding another op,
1800091d81d1SSam Leffler 					 * regardless whether its for the same
1801091d81d1SSam Leffler 					 * driver or not.  We could keep
1802091d81d1SSam Leffler 					 * searching the queue but it might be
1803091d81d1SSam Leffler 					 * better to just use a per-driver
1804091d81d1SSam Leffler 					 * queue instead.
1805091d81d1SSam Leffler 					 */
1806c0341432SJohn Baldwin 					if (submit->crp_session->cap == cap)
1807091d81d1SSam Leffler 						hint = CRYPTO_HINT_MORE;
1808091d81d1SSam Leffler 				} else {
1809091d81d1SSam Leffler 					submit = crp;
1810091d81d1SSam Leffler 				}
181168f6800cSMark Johnston 				break;
1812091d81d1SSam Leffler 			}
1813091d81d1SSam Leffler 		}
1814091d81d1SSam Leffler 		if (submit != NULL) {
1815091d81d1SSam Leffler 			TAILQ_REMOVE(&crp_q, submit, crp_next);
1816c0341432SJohn Baldwin 			cap = submit->crp_session->cap;
1817c3c82036SPawel Jakub Dawidek 			KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1818c3c82036SPawel Jakub Dawidek 			    __func__, __LINE__));
1819c0341432SJohn Baldwin 			CRYPTO_Q_UNLOCK();
18204acae0acSPawel Jakub Dawidek 			result = crypto_invoke(cap, submit, hint);
1821c0341432SJohn Baldwin 			CRYPTO_Q_LOCK();
1822091d81d1SSam Leffler 			if (result == ERESTART) {
1823091d81d1SSam Leffler 				/*
1824091d81d1SSam Leffler 				 * The driver ran out of resources, mark the
1825091d81d1SSam Leffler 				 * driver ``blocked'' for cryptop's and put
1826091d81d1SSam Leffler 				 * the request back in the queue.  It would
1827091d81d1SSam Leffler 				 * best to put the request back where we got
1828091d81d1SSam Leffler 				 * it but that's hard so for now we put it
1829091d81d1SSam Leffler 				 * at the front.  This should be ok; putting
1830091d81d1SSam Leffler 				 * it at the end does not work.
1831091d81d1SSam Leffler 				 */
1832c0341432SJohn Baldwin 				cap->cc_qblocked = 1;
1833091d81d1SSam Leffler 				TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
18347290cb47SMark Johnston 				CRYPTOSTAT_INC(cs_blocks);
1835091d81d1SSam Leffler 			}
183676681661SJohn Baldwin 		} else {
1837091d81d1SSam Leffler 			/*
1838091d81d1SSam Leffler 			 * Nothing more to be processed.  Sleep until we're
1839091d81d1SSam Leffler 			 * woken because there are more ops to process.
1840091d81d1SSam Leffler 			 * This happens either by submission or by a driver
1841091d81d1SSam Leffler 			 * becoming unblocked and notifying us through
1842091d81d1SSam Leffler 			 * crypto_unblock.  Note that when we wakeup we
1843091d81d1SSam Leffler 			 * start processing each queue again from the
1844091d81d1SSam Leffler 			 * front. It's not clear that it's important to
1845091d81d1SSam Leffler 			 * preserve this ordering since ops may finish
1846091d81d1SSam Leffler 			 * out of order if dispatched to different devices
1847091d81d1SSam Leffler 			 * and some become blocked while others do not.
1848091d81d1SSam Leffler 			 */
18493a865c82SPawel Jakub Dawidek 			crp_sleep = 1;
18501a91ccccSSam Leffler 			msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
18513a865c82SPawel Jakub Dawidek 			crp_sleep = 0;
185271785781SJohn Baldwin 			if (cryptotd == NULL)
185351e45326SSam Leffler 				break;
18547290cb47SMark Johnston 			CRYPTOSTAT_INC(cs_intrs);
1855091d81d1SSam Leffler 		}
1856091d81d1SSam Leffler 	}
185751e45326SSam Leffler 	CRYPTO_Q_UNLOCK();
18581a91ccccSSam Leffler 
185951e45326SSam Leffler 	crypto_finis(&crp_q);
18601a91ccccSSam Leffler }
18611a91ccccSSam Leffler 
18621a91ccccSSam Leffler /*
18631a91ccccSSam Leffler  * Crypto returns thread, does callbacks for processed crypto requests.
18641a91ccccSSam Leffler  * Callbacks are done here, rather than in the crypto drivers, because
18651a91ccccSSam Leffler  * callbacks typically are expensive and would slow interrupt handling.
18661a91ccccSSam Leffler  */
18671a91ccccSSam Leffler static void
186871785781SJohn Baldwin crypto_ret_thread(void *arg)
18691a91ccccSSam Leffler {
187071785781SJohn Baldwin 	struct crypto_ret_worker *ret_worker = arg;
18711a91ccccSSam Leffler 	struct cryptop *crpt;
18721a91ccccSSam Leffler 
187339bbca6fSFabien Thomas 	CRYPTO_RETW_LOCK(ret_worker);
18741a91ccccSSam Leffler 	for (;;) {
18751a91ccccSSam Leffler 		/* Harvest return q's for completed ops */
187639bbca6fSFabien Thomas 		crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q);
187739bbca6fSFabien Thomas 		if (crpt != NULL) {
187839bbca6fSFabien Thomas 			if (crpt->crp_seq == ret_worker->reorder_cur_seq) {
187939bbca6fSFabien Thomas 				TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next);
188039bbca6fSFabien Thomas 				ret_worker->reorder_cur_seq++;
188139bbca6fSFabien Thomas 			} else {
188239bbca6fSFabien Thomas 				crpt = NULL;
188339bbca6fSFabien Thomas 			}
188439bbca6fSFabien Thomas 		}
18851a91ccccSSam Leffler 
188639bbca6fSFabien Thomas 		if (crpt == NULL) {
188739bbca6fSFabien Thomas 			crpt = TAILQ_FIRST(&ret_worker->crp_ret_q);
188839bbca6fSFabien Thomas 			if (crpt != NULL)
188939bbca6fSFabien Thomas 				TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next);
189039bbca6fSFabien Thomas 		}
189139bbca6fSFabien Thomas 
189276681661SJohn Baldwin 		if (crpt != NULL) {
189339bbca6fSFabien Thomas 			CRYPTO_RETW_UNLOCK(ret_worker);
18941a91ccccSSam Leffler 			/*
18951a91ccccSSam Leffler 			 * Run callbacks unlocked.
18961a91ccccSSam Leffler 			 */
1897a5c053f5SMark Johnston 			if (crpt != NULL)
18981a91ccccSSam Leffler 				crpt->crp_callback(crpt);
189939bbca6fSFabien Thomas 			CRYPTO_RETW_LOCK(ret_worker);
19001a91ccccSSam Leffler 		} else {
19011a91ccccSSam Leffler 			/*
19021a91ccccSSam Leffler 			 * Nothing more to be processed.  Sleep until we're
19031a91ccccSSam Leffler 			 * woken because there are more returns to process.
19041a91ccccSSam Leffler 			 */
190539bbca6fSFabien Thomas 			msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,
19061a91ccccSSam Leffler 				"crypto_ret_wait", 0);
190771785781SJohn Baldwin 			if (ret_worker->td == NULL)
190851e45326SSam Leffler 				break;
19097290cb47SMark Johnston 			CRYPTOSTAT_INC(cs_rets);
19101a91ccccSSam Leffler 		}
19111a91ccccSSam Leffler 	}
191239bbca6fSFabien Thomas 	CRYPTO_RETW_UNLOCK(ret_worker);
191351e45326SSam Leffler 
191439bbca6fSFabien Thomas 	crypto_finis(&ret_worker->crp_ret_q);
19151a91ccccSSam Leffler }
19166810ad6fSSam Leffler 
19176810ad6fSSam Leffler #ifdef DDB
19186810ad6fSSam Leffler static void
19196810ad6fSSam Leffler db_show_drivers(void)
19206810ad6fSSam Leffler {
19216810ad6fSSam Leffler 	int hid;
19226810ad6fSSam Leffler 
192376681661SJohn Baldwin 	db_printf("%12s %4s %8s %2s\n"
19246810ad6fSSam Leffler 		, "Device"
19256810ad6fSSam Leffler 		, "Ses"
19266810ad6fSSam Leffler 		, "Flags"
19276810ad6fSSam Leffler 		, "QB"
19286810ad6fSSam Leffler 	);
1929c0341432SJohn Baldwin 	for (hid = 0; hid < crypto_drivers_size; hid++) {
1930c0341432SJohn Baldwin 		const struct cryptocap *cap = crypto_drivers[hid];
1931c0341432SJohn Baldwin 		if (cap == NULL)
19326810ad6fSSam Leffler 			continue;
193376681661SJohn Baldwin 		db_printf("%-12s %4u %08x %2u\n"
19346810ad6fSSam Leffler 		    , device_get_nameunit(cap->cc_dev)
19356810ad6fSSam Leffler 		    , cap->cc_sessions
19366810ad6fSSam Leffler 		    , cap->cc_flags
19376810ad6fSSam Leffler 		    , cap->cc_qblocked
19386810ad6fSSam Leffler 		);
19396810ad6fSSam Leffler 	}
19406810ad6fSSam Leffler }
19416810ad6fSSam Leffler 
19426810ad6fSSam Leffler DB_SHOW_COMMAND(crypto, db_show_crypto)
19436810ad6fSSam Leffler {
19446810ad6fSSam Leffler 	struct cryptop *crp;
194539bbca6fSFabien Thomas 	struct crypto_ret_worker *ret_worker;
19466810ad6fSSam Leffler 
19476810ad6fSSam Leffler 	db_show_drivers();
19486810ad6fSSam Leffler 	db_printf("\n");
19496810ad6fSSam Leffler 
19506810ad6fSSam Leffler 	db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
19516810ad6fSSam Leffler 	    "HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
1952c0341432SJohn Baldwin 	    "Device", "Callback");
19536810ad6fSSam Leffler 	TAILQ_FOREACH(crp, &crp_q, crp_next) {
19549c0e3d3aSJohn Baldwin 		db_printf("%4u %08x %4u %4u %04x %8p %8p\n"
1955c0341432SJohn Baldwin 		    , crp->crp_session->cap->cc_hid
19561b0909d5SConrad Meyer 		    , (int) crypto_ses2caps(crp->crp_session)
19579c0e3d3aSJohn Baldwin 		    , crp->crp_olen
19586810ad6fSSam Leffler 		    , crp->crp_etype
19596810ad6fSSam Leffler 		    , crp->crp_flags
1960c0341432SJohn Baldwin 		    , device_get_nameunit(crp->crp_session->cap->cc_dev)
19616810ad6fSSam Leffler 		    , crp->crp_callback
19626810ad6fSSam Leffler 		);
19636810ad6fSSam Leffler 	}
196439bbca6fSFabien Thomas 	FOREACH_CRYPTO_RETW(ret_worker) {
196539bbca6fSFabien Thomas 		db_printf("\n%8s %4s %4s %4s %8s\n",
196639bbca6fSFabien Thomas 		    "ret_worker", "HID", "Etype", "Flags", "Callback");
196739bbca6fSFabien Thomas 		if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
196839bbca6fSFabien Thomas 			TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
196939bbca6fSFabien Thomas 				db_printf("%8td %4u %4u %04x %8p\n"
197039bbca6fSFabien Thomas 				    , CRYPTO_RETW_ID(ret_worker)
1971c0341432SJohn Baldwin 				    , crp->crp_session->cap->cc_hid
19726810ad6fSSam Leffler 				    , crp->crp_etype
19736810ad6fSSam Leffler 				    , crp->crp_flags
19746810ad6fSSam Leffler 				    , crp->crp_callback
19756810ad6fSSam Leffler 				);
19766810ad6fSSam Leffler 			}
19776810ad6fSSam Leffler 		}
19786810ad6fSSam Leffler 	}
197939bbca6fSFabien Thomas }
19806810ad6fSSam Leffler #endif
19816810ad6fSSam Leffler 
19826810ad6fSSam Leffler int crypto_modevent(module_t mod, int type, void *unused);
19836810ad6fSSam Leffler 
19846810ad6fSSam Leffler /*
19856810ad6fSSam Leffler  * Initialization code, both for static and dynamic loading.
19866810ad6fSSam Leffler  * Note this is not invoked with the usual MODULE_DECLARE
19876810ad6fSSam Leffler  * mechanism but instead is listed as a dependency by the
19886810ad6fSSam Leffler  * cryptosoft driver.  This guarantees proper ordering of
19896810ad6fSSam Leffler  * calls on module load/unload.
19906810ad6fSSam Leffler  */
19916810ad6fSSam Leffler int
19926810ad6fSSam Leffler crypto_modevent(module_t mod, int type, void *unused)
19936810ad6fSSam Leffler {
19946810ad6fSSam Leffler 	int error = EINVAL;
19956810ad6fSSam Leffler 
19966810ad6fSSam Leffler 	switch (type) {
19976810ad6fSSam Leffler 	case MOD_LOAD:
19986810ad6fSSam Leffler 		error = crypto_init();
19996810ad6fSSam Leffler 		if (error == 0 && bootverbose)
20006810ad6fSSam Leffler 			printf("crypto: <crypto core>\n");
20016810ad6fSSam Leffler 		break;
20026810ad6fSSam Leffler 	case MOD_UNLOAD:
20036810ad6fSSam Leffler 		/*XXX disallow if active sessions */
20046810ad6fSSam Leffler 		error = 0;
20056810ad6fSSam Leffler 		crypto_destroy();
20066810ad6fSSam Leffler 		return 0;
20076810ad6fSSam Leffler 	}
20086810ad6fSSam Leffler 	return error;
20096810ad6fSSam Leffler }
20106810ad6fSSam Leffler MODULE_VERSION(crypto, 1);
20116810ad6fSSam Leffler MODULE_DEPEND(crypto, zlib, 1, 1, 1);
2012