xref: /freebsd/sys/opencrypto/ktls_ocf.c (revision 55b7a0e1e7784738f7128c00edc719064b4bc921)
1b2e60773SJohn Baldwin /*-
2b2e60773SJohn Baldwin  * SPDX-License-Identifier: BSD-2-Clause
3b2e60773SJohn Baldwin  *
4b2e60773SJohn Baldwin  * Copyright (c) 2019 Netflix Inc.
5b2e60773SJohn Baldwin  * All rights reserved.
6b2e60773SJohn Baldwin  *
7b2e60773SJohn Baldwin  * Redistribution and use in source and binary forms, with or without
8b2e60773SJohn Baldwin  * modification, are permitted provided that the following conditions
9b2e60773SJohn Baldwin  * are met:
10b2e60773SJohn Baldwin  * 1. Redistributions of source code must retain the above copyright
11b2e60773SJohn Baldwin  *    notice, this list of conditions and the following disclaimer.
12b2e60773SJohn Baldwin  * 2. Redistributions in binary form must reproduce the above copyright
13b2e60773SJohn Baldwin  *    notice, this list of conditions and the following disclaimer in the
14b2e60773SJohn Baldwin  *    documentation and/or other materials provided with the distribution.
15b2e60773SJohn Baldwin  *
16b2e60773SJohn Baldwin  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17b2e60773SJohn Baldwin  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18b2e60773SJohn Baldwin  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19b2e60773SJohn Baldwin  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
20b2e60773SJohn Baldwin  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21b2e60773SJohn Baldwin  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22b2e60773SJohn Baldwin  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23b2e60773SJohn Baldwin  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24b2e60773SJohn Baldwin  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25b2e60773SJohn Baldwin  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26b2e60773SJohn Baldwin  * SUCH DAMAGE.
27b2e60773SJohn Baldwin  */
28b2e60773SJohn Baldwin 
29b2e60773SJohn Baldwin #include <sys/cdefs.h>
30b2e60773SJohn Baldwin __FBSDID("$FreeBSD$");
31b2e60773SJohn Baldwin 
32b2e60773SJohn Baldwin #include <sys/param.h>
33b2e60773SJohn Baldwin #include <sys/systm.h>
34b2e60773SJohn Baldwin #include <sys/counter.h>
35b2e60773SJohn Baldwin #include <sys/endian.h>
36b2e60773SJohn Baldwin #include <sys/kernel.h>
37b2e60773SJohn Baldwin #include <sys/ktls.h>
38b2e60773SJohn Baldwin #include <sys/lock.h>
39b2e60773SJohn Baldwin #include <sys/malloc.h>
40b2e60773SJohn Baldwin #include <sys/module.h>
41b2e60773SJohn Baldwin #include <sys/mutex.h>
42b2e60773SJohn Baldwin #include <sys/sysctl.h>
43b2e60773SJohn Baldwin #include <sys/uio.h>
44b2e60773SJohn Baldwin #include <opencrypto/cryptodev.h>
45b2e60773SJohn Baldwin 
46b2e60773SJohn Baldwin struct ocf_session {
47b2e60773SJohn Baldwin 	crypto_session_t sid;
48b2e60773SJohn Baldwin 	int crda_alg;
49b2e60773SJohn Baldwin 	struct mtx lock;
50b2e60773SJohn Baldwin };
51b2e60773SJohn Baldwin 
52b2e60773SJohn Baldwin struct ocf_operation {
53b2e60773SJohn Baldwin 	struct ocf_session *os;
54b2e60773SJohn Baldwin 	bool done;
55b2e60773SJohn Baldwin 	struct iovec iov[0];
56b2e60773SJohn Baldwin };
57b2e60773SJohn Baldwin 
58b2e60773SJohn Baldwin static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS");
59b2e60773SJohn Baldwin 
60b2e60773SJohn Baldwin SYSCTL_DECL(_kern_ipc_tls);
61b2e60773SJohn Baldwin SYSCTL_DECL(_kern_ipc_tls_stats);
62b2e60773SJohn Baldwin 
63*55b7a0e1SJohn Baldwin static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf, CTLFLAG_RD, 0,
64*55b7a0e1SJohn Baldwin     "Kernel TLS offload via OCF stats");
65*55b7a0e1SJohn Baldwin 
66*55b7a0e1SJohn Baldwin static counter_u64_t ocf_tls12_gcm_crypts;
67*55b7a0e1SJohn Baldwin SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts,
68*55b7a0e1SJohn Baldwin     CTLFLAG_RD, &ocf_tls12_gcm_crypts,
69*55b7a0e1SJohn Baldwin     "Total number of OCF TLS 1.2 GCM encryption operations");
70*55b7a0e1SJohn Baldwin 
71*55b7a0e1SJohn Baldwin static counter_u64_t ocf_tls13_gcm_crypts;
72*55b7a0e1SJohn Baldwin SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts,
73*55b7a0e1SJohn Baldwin     CTLFLAG_RD, &ocf_tls13_gcm_crypts,
74*55b7a0e1SJohn Baldwin     "Total number of OCF TLS 1.3 GCM encryption operations");
75b2e60773SJohn Baldwin 
76b2e60773SJohn Baldwin static counter_u64_t ocf_retries;
77*55b7a0e1SJohn Baldwin SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD,
78b2e60773SJohn Baldwin     &ocf_retries,
79b2e60773SJohn Baldwin     "Number of OCF encryption operation retries");
80b2e60773SJohn Baldwin 
81b2e60773SJohn Baldwin static int
82b2e60773SJohn Baldwin ktls_ocf_callback(struct cryptop *crp)
83b2e60773SJohn Baldwin {
84b2e60773SJohn Baldwin 	struct ocf_operation *oo;
85b2e60773SJohn Baldwin 
86b2e60773SJohn Baldwin 	oo = crp->crp_opaque;
87b2e60773SJohn Baldwin 	mtx_lock(&oo->os->lock);
88b2e60773SJohn Baldwin 	oo->done = true;
89b2e60773SJohn Baldwin 	mtx_unlock(&oo->os->lock);
90b2e60773SJohn Baldwin 	wakeup(oo);
91b2e60773SJohn Baldwin 	return (0);
92b2e60773SJohn Baldwin }
93b2e60773SJohn Baldwin 
94b2e60773SJohn Baldwin static int
95*55b7a0e1SJohn Baldwin ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls,
96*55b7a0e1SJohn Baldwin     const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
97*55b7a0e1SJohn Baldwin     struct iovec *outiov, int iovcnt, uint64_t seqno,
98*55b7a0e1SJohn Baldwin     uint8_t record_type __unused)
99b2e60773SJohn Baldwin {
100b2e60773SJohn Baldwin 	struct uio uio;
101b2e60773SJohn Baldwin 	struct tls_aead_data ad;
102b2e60773SJohn Baldwin 	struct tls_nonce_data nd;
103b2e60773SJohn Baldwin 	struct cryptodesc *crde, *crda;
104b2e60773SJohn Baldwin 	struct cryptop *crp;
105b2e60773SJohn Baldwin 	struct ocf_session *os;
106b2e60773SJohn Baldwin 	struct ocf_operation *oo;
107b2e60773SJohn Baldwin 	struct iovec *iov;
108b2e60773SJohn Baldwin 	int i, error;
109b2e60773SJohn Baldwin 	uint16_t tls_comp_len;
110b2e60773SJohn Baldwin 
111b2e60773SJohn Baldwin 	os = tls->cipher;
112b2e60773SJohn Baldwin 
113b2e60773SJohn Baldwin 	oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF,
114b2e60773SJohn Baldwin 	    M_WAITOK | M_ZERO);
115b2e60773SJohn Baldwin 	oo->os = os;
116b2e60773SJohn Baldwin 	iov = oo->iov;
117b2e60773SJohn Baldwin 
118b2e60773SJohn Baldwin 	crp = crypto_getreq(2);
119b2e60773SJohn Baldwin 	if (crp == NULL) {
120b2e60773SJohn Baldwin 		free(oo, M_KTLS_OCF);
121b2e60773SJohn Baldwin 		return (ENOMEM);
122b2e60773SJohn Baldwin 	}
123b2e60773SJohn Baldwin 
124b2e60773SJohn Baldwin 	/* Setup the IV. */
125b2e60773SJohn Baldwin 	memcpy(nd.fixed, tls->params.iv, TLS_AEAD_GCM_LEN);
126b2e60773SJohn Baldwin 	memcpy(&nd.seq, hdr + 1, sizeof(nd.seq));
127b2e60773SJohn Baldwin 
128b2e60773SJohn Baldwin 	/* Setup the AAD. */
129b2e60773SJohn Baldwin 	tls_comp_len = ntohs(hdr->tls_length) -
130b2e60773SJohn Baldwin 	    (AES_GMAC_HASH_LEN + sizeof(nd.seq));
131b2e60773SJohn Baldwin 	ad.seq = htobe64(seqno);
132b2e60773SJohn Baldwin 	ad.type = hdr->tls_type;
133b2e60773SJohn Baldwin 	ad.tls_vmajor = hdr->tls_vmajor;
134b2e60773SJohn Baldwin 	ad.tls_vminor = hdr->tls_vminor;
135b2e60773SJohn Baldwin 	ad.tls_length = htons(tls_comp_len);
136b2e60773SJohn Baldwin 	iov[0].iov_base = &ad;
137b2e60773SJohn Baldwin 	iov[0].iov_len = sizeof(ad);
138b2e60773SJohn Baldwin 	uio.uio_resid = sizeof(ad);
139b2e60773SJohn Baldwin 
140b2e60773SJohn Baldwin 	/*
141b2e60773SJohn Baldwin 	 * OCF always does encryption in place, so copy the data if
142b2e60773SJohn Baldwin 	 * needed.  Ugh.
143b2e60773SJohn Baldwin 	 */
144b2e60773SJohn Baldwin 	for (i = 0; i < iovcnt; i++) {
145b2e60773SJohn Baldwin 		iov[i + 1] = outiov[i];
146b2e60773SJohn Baldwin 		if (iniov[i].iov_base != outiov[i].iov_base)
147b2e60773SJohn Baldwin 			memcpy(outiov[i].iov_base, iniov[i].iov_base,
148b2e60773SJohn Baldwin 			    outiov[i].iov_len);
149b2e60773SJohn Baldwin 		uio.uio_resid += outiov[i].iov_len;
150b2e60773SJohn Baldwin 	}
151b2e60773SJohn Baldwin 
152b2e60773SJohn Baldwin 	iov[iovcnt + 1].iov_base = trailer;
153b2e60773SJohn Baldwin 	iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN;
154b2e60773SJohn Baldwin 	uio.uio_resid += AES_GMAC_HASH_LEN;
155b2e60773SJohn Baldwin 
156b2e60773SJohn Baldwin 	uio.uio_iov = iov;
157b2e60773SJohn Baldwin 	uio.uio_iovcnt = iovcnt + 2;
158b2e60773SJohn Baldwin 	uio.uio_offset = 0;
159b2e60773SJohn Baldwin 	uio.uio_segflg = UIO_SYSSPACE;
160b2e60773SJohn Baldwin 	uio.uio_td = curthread;
161b2e60773SJohn Baldwin 
162b2e60773SJohn Baldwin 	crp->crp_session = os->sid;
163b2e60773SJohn Baldwin 	crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM;
164b2e60773SJohn Baldwin 	crp->crp_uio = &uio;
165b2e60773SJohn Baldwin 	crp->crp_ilen = uio.uio_resid;
166b2e60773SJohn Baldwin 	crp->crp_opaque = oo;
167b2e60773SJohn Baldwin 	crp->crp_callback = ktls_ocf_callback;
168b2e60773SJohn Baldwin 
169b2e60773SJohn Baldwin 	crde = crp->crp_desc;
170b2e60773SJohn Baldwin 	crda = crde->crd_next;
171b2e60773SJohn Baldwin 
172b2e60773SJohn Baldwin 	crda->crd_alg = os->crda_alg;
173b2e60773SJohn Baldwin 	crda->crd_skip = 0;
174b2e60773SJohn Baldwin 	crda->crd_len = sizeof(ad);
175b2e60773SJohn Baldwin 	crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN;
176b2e60773SJohn Baldwin 
177b2e60773SJohn Baldwin 	crde->crd_alg = CRYPTO_AES_NIST_GCM_16;
178b2e60773SJohn Baldwin 	crde->crd_skip = sizeof(ad);
179b2e60773SJohn Baldwin 	crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN);
180b2e60773SJohn Baldwin 	crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
181b2e60773SJohn Baldwin 	memcpy(crde->crd_iv, &nd, sizeof(nd));
182b2e60773SJohn Baldwin 
183*55b7a0e1SJohn Baldwin 	counter_u64_add(ocf_tls12_gcm_crypts, 1);
184*55b7a0e1SJohn Baldwin 	for (;;) {
185*55b7a0e1SJohn Baldwin 		error = crypto_dispatch(crp);
186*55b7a0e1SJohn Baldwin 		if (error)
187*55b7a0e1SJohn Baldwin 			break;
188*55b7a0e1SJohn Baldwin 
189*55b7a0e1SJohn Baldwin 		mtx_lock(&os->lock);
190*55b7a0e1SJohn Baldwin 		while (!oo->done)
191*55b7a0e1SJohn Baldwin 			mtx_sleep(oo, &os->lock, 0, "ocfktls", 0);
192*55b7a0e1SJohn Baldwin 		mtx_unlock(&os->lock);
193*55b7a0e1SJohn Baldwin 
194*55b7a0e1SJohn Baldwin 		if (crp->crp_etype != EAGAIN) {
195*55b7a0e1SJohn Baldwin 			error = crp->crp_etype;
196*55b7a0e1SJohn Baldwin 			break;
197*55b7a0e1SJohn Baldwin 		}
198*55b7a0e1SJohn Baldwin 
199*55b7a0e1SJohn Baldwin 		crp->crp_etype = 0;
200*55b7a0e1SJohn Baldwin 		crp->crp_flags &= ~CRYPTO_F_DONE;
201*55b7a0e1SJohn Baldwin 		oo->done = false;
202*55b7a0e1SJohn Baldwin 		counter_u64_add(ocf_retries, 1);
203*55b7a0e1SJohn Baldwin 	}
204*55b7a0e1SJohn Baldwin 
205*55b7a0e1SJohn Baldwin 	crypto_freereq(crp);
206*55b7a0e1SJohn Baldwin 	free(oo, M_KTLS_OCF);
207*55b7a0e1SJohn Baldwin 	return (error);
208*55b7a0e1SJohn Baldwin }
209*55b7a0e1SJohn Baldwin 
210*55b7a0e1SJohn Baldwin static int
211*55b7a0e1SJohn Baldwin ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls,
212*55b7a0e1SJohn Baldwin     const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
213*55b7a0e1SJohn Baldwin     struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type)
214*55b7a0e1SJohn Baldwin {
215*55b7a0e1SJohn Baldwin 	struct uio uio;
216*55b7a0e1SJohn Baldwin 	struct tls_aead_data_13 ad;
217*55b7a0e1SJohn Baldwin 	char nonce[12];
218*55b7a0e1SJohn Baldwin 	struct cryptodesc *crde, *crda;
219*55b7a0e1SJohn Baldwin 	struct cryptop *crp;
220*55b7a0e1SJohn Baldwin 	struct ocf_session *os;
221*55b7a0e1SJohn Baldwin 	struct ocf_operation *oo;
222*55b7a0e1SJohn Baldwin 	struct iovec *iov;
223*55b7a0e1SJohn Baldwin 	int i, error;
224*55b7a0e1SJohn Baldwin 
225*55b7a0e1SJohn Baldwin 	os = tls->cipher;
226*55b7a0e1SJohn Baldwin 
227*55b7a0e1SJohn Baldwin 	oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF,
228*55b7a0e1SJohn Baldwin 	    M_WAITOK | M_ZERO);
229*55b7a0e1SJohn Baldwin 	oo->os = os;
230*55b7a0e1SJohn Baldwin 	iov = oo->iov;
231*55b7a0e1SJohn Baldwin 
232*55b7a0e1SJohn Baldwin 	crp = crypto_getreq(2);
233*55b7a0e1SJohn Baldwin 	if (crp == NULL) {
234*55b7a0e1SJohn Baldwin 		free(oo, M_KTLS_OCF);
235*55b7a0e1SJohn Baldwin 		return (ENOMEM);
236*55b7a0e1SJohn Baldwin 	}
237*55b7a0e1SJohn Baldwin 
238*55b7a0e1SJohn Baldwin 	/* Setup the nonce. */
239*55b7a0e1SJohn Baldwin 	memcpy(nonce, tls->params.iv, tls->params.iv_len);
240*55b7a0e1SJohn Baldwin 	*(uint64_t *)(nonce + 4) ^= htobe64(seqno);
241*55b7a0e1SJohn Baldwin 
242*55b7a0e1SJohn Baldwin 	/* Setup the AAD. */
243*55b7a0e1SJohn Baldwin 	ad.type = hdr->tls_type;
244*55b7a0e1SJohn Baldwin 	ad.tls_vmajor = hdr->tls_vmajor;
245*55b7a0e1SJohn Baldwin 	ad.tls_vminor = hdr->tls_vminor;
246*55b7a0e1SJohn Baldwin 	ad.tls_length = hdr->tls_length;
247*55b7a0e1SJohn Baldwin 	iov[0].iov_base = &ad;
248*55b7a0e1SJohn Baldwin 	iov[0].iov_len = sizeof(ad);
249*55b7a0e1SJohn Baldwin 	uio.uio_resid = sizeof(ad);
250*55b7a0e1SJohn Baldwin 
251*55b7a0e1SJohn Baldwin 	/*
252*55b7a0e1SJohn Baldwin 	 * OCF always does encryption in place, so copy the data if
253*55b7a0e1SJohn Baldwin 	 * needed.  Ugh.
254*55b7a0e1SJohn Baldwin 	 */
255*55b7a0e1SJohn Baldwin 	for (i = 0; i < iovcnt; i++) {
256*55b7a0e1SJohn Baldwin 		iov[i + 1] = outiov[i];
257*55b7a0e1SJohn Baldwin 		if (iniov[i].iov_base != outiov[i].iov_base)
258*55b7a0e1SJohn Baldwin 			memcpy(outiov[i].iov_base, iniov[i].iov_base,
259*55b7a0e1SJohn Baldwin 			    outiov[i].iov_len);
260*55b7a0e1SJohn Baldwin 		uio.uio_resid += outiov[i].iov_len;
261*55b7a0e1SJohn Baldwin 	}
262*55b7a0e1SJohn Baldwin 
263*55b7a0e1SJohn Baldwin 	trailer[0] = record_type;
264*55b7a0e1SJohn Baldwin 	iov[iovcnt + 1].iov_base = trailer;
265*55b7a0e1SJohn Baldwin 	iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN + 1;
266*55b7a0e1SJohn Baldwin 	uio.uio_resid += AES_GMAC_HASH_LEN + 1;
267*55b7a0e1SJohn Baldwin 
268*55b7a0e1SJohn Baldwin 	uio.uio_iov = iov;
269*55b7a0e1SJohn Baldwin 	uio.uio_iovcnt = iovcnt + 2;
270*55b7a0e1SJohn Baldwin 	uio.uio_offset = 0;
271*55b7a0e1SJohn Baldwin 	uio.uio_segflg = UIO_SYSSPACE;
272*55b7a0e1SJohn Baldwin 	uio.uio_td = curthread;
273*55b7a0e1SJohn Baldwin 
274*55b7a0e1SJohn Baldwin 	crp->crp_session = os->sid;
275*55b7a0e1SJohn Baldwin 	crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM;
276*55b7a0e1SJohn Baldwin 	crp->crp_uio = &uio;
277*55b7a0e1SJohn Baldwin 	crp->crp_ilen = uio.uio_resid;
278*55b7a0e1SJohn Baldwin 	crp->crp_opaque = oo;
279*55b7a0e1SJohn Baldwin 	crp->crp_callback = ktls_ocf_callback;
280*55b7a0e1SJohn Baldwin 
281*55b7a0e1SJohn Baldwin 	crde = crp->crp_desc;
282*55b7a0e1SJohn Baldwin 	crda = crde->crd_next;
283*55b7a0e1SJohn Baldwin 
284*55b7a0e1SJohn Baldwin 	crda->crd_alg = os->crda_alg;
285*55b7a0e1SJohn Baldwin 	crda->crd_skip = 0;
286*55b7a0e1SJohn Baldwin 	crda->crd_len = sizeof(ad);
287*55b7a0e1SJohn Baldwin 	crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN;
288*55b7a0e1SJohn Baldwin 
289*55b7a0e1SJohn Baldwin 	crde->crd_alg = CRYPTO_AES_NIST_GCM_16;
290*55b7a0e1SJohn Baldwin 	crde->crd_skip = sizeof(ad);
291*55b7a0e1SJohn Baldwin 	crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN);
292*55b7a0e1SJohn Baldwin 	crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
293*55b7a0e1SJohn Baldwin 	memcpy(crde->crd_iv, nonce, sizeof(nonce));
294*55b7a0e1SJohn Baldwin 
295*55b7a0e1SJohn Baldwin 	counter_u64_add(ocf_tls13_gcm_crypts, 1);
296b2e60773SJohn Baldwin 	for (;;) {
297b2e60773SJohn Baldwin 		error = crypto_dispatch(crp);
298b2e60773SJohn Baldwin 		if (error)
299b2e60773SJohn Baldwin 			break;
300b2e60773SJohn Baldwin 
301b2e60773SJohn Baldwin 		mtx_lock(&os->lock);
302b2e60773SJohn Baldwin 		while (!oo->done)
303b2e60773SJohn Baldwin 			mtx_sleep(oo, &os->lock, 0, "ocfktls", 0);
304b2e60773SJohn Baldwin 		mtx_unlock(&os->lock);
305b2e60773SJohn Baldwin 
306b2e60773SJohn Baldwin 		if (crp->crp_etype != EAGAIN) {
307b2e60773SJohn Baldwin 			error = crp->crp_etype;
308b2e60773SJohn Baldwin 			break;
309b2e60773SJohn Baldwin 		}
310b2e60773SJohn Baldwin 
311b2e60773SJohn Baldwin 		crp->crp_etype = 0;
312b2e60773SJohn Baldwin 		crp->crp_flags &= ~CRYPTO_F_DONE;
313b2e60773SJohn Baldwin 		oo->done = false;
314b2e60773SJohn Baldwin 		counter_u64_add(ocf_retries, 1);
315b2e60773SJohn Baldwin 	}
316b2e60773SJohn Baldwin 
317b2e60773SJohn Baldwin 	crypto_freereq(crp);
318b2e60773SJohn Baldwin 	free(oo, M_KTLS_OCF);
319b2e60773SJohn Baldwin 	return (error);
320b2e60773SJohn Baldwin }
321b2e60773SJohn Baldwin 
322b2e60773SJohn Baldwin static void
323b2e60773SJohn Baldwin ktls_ocf_free(struct ktls_session *tls)
324b2e60773SJohn Baldwin {
325b2e60773SJohn Baldwin 	struct ocf_session *os;
326b2e60773SJohn Baldwin 
327b2e60773SJohn Baldwin 	os = tls->cipher;
328b2e60773SJohn Baldwin 	mtx_destroy(&os->lock);
329b2e60773SJohn Baldwin 	explicit_bzero(os, sizeof(*os));
330b2e60773SJohn Baldwin 	free(os, M_KTLS_OCF);
331b2e60773SJohn Baldwin }
332b2e60773SJohn Baldwin 
333b2e60773SJohn Baldwin static int
334b2e60773SJohn Baldwin ktls_ocf_try(struct socket *so, struct ktls_session *tls)
335b2e60773SJohn Baldwin {
336b2e60773SJohn Baldwin 	struct cryptoini cria, crie;
337b2e60773SJohn Baldwin 	struct ocf_session *os;
338b2e60773SJohn Baldwin 	int error;
339b2e60773SJohn Baldwin 
340b2e60773SJohn Baldwin 	memset(&cria, 0, sizeof(cria));
341b2e60773SJohn Baldwin 	memset(&crie, 0, sizeof(crie));
342b2e60773SJohn Baldwin 
343b2e60773SJohn Baldwin 	switch (tls->params.cipher_algorithm) {
344b2e60773SJohn Baldwin 	case CRYPTO_AES_NIST_GCM_16:
345b2e60773SJohn Baldwin 		switch (tls->params.cipher_key_len) {
346b2e60773SJohn Baldwin 		case 128 / 8:
347b2e60773SJohn Baldwin 			cria.cri_alg = CRYPTO_AES_128_NIST_GMAC;
348b2e60773SJohn Baldwin 			break;
349b2e60773SJohn Baldwin 		case 256 / 8:
350b2e60773SJohn Baldwin 			cria.cri_alg = CRYPTO_AES_256_NIST_GMAC;
351b2e60773SJohn Baldwin 			break;
352b2e60773SJohn Baldwin 		default:
353b2e60773SJohn Baldwin 			return (EINVAL);
354b2e60773SJohn Baldwin 		}
355b2e60773SJohn Baldwin 		cria.cri_key = tls->params.cipher_key;
356b2e60773SJohn Baldwin 		cria.cri_klen = tls->params.cipher_key_len * 8;
357b2e60773SJohn Baldwin 		break;
358b2e60773SJohn Baldwin 	default:
359b2e60773SJohn Baldwin 		return (EPROTONOSUPPORT);
360b2e60773SJohn Baldwin 	}
361b2e60773SJohn Baldwin 
362*55b7a0e1SJohn Baldwin 	/* Only TLS 1.2 and 1.3 are supported. */
363b2e60773SJohn Baldwin 	if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
364*55b7a0e1SJohn Baldwin 	    tls->params.tls_vminor < TLS_MINOR_VER_TWO ||
365*55b7a0e1SJohn Baldwin 	    tls->params.tls_vminor > TLS_MINOR_VER_THREE)
366b2e60773SJohn Baldwin 		return (EPROTONOSUPPORT);
367b2e60773SJohn Baldwin 
368b2e60773SJohn Baldwin 	os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO);
369b2e60773SJohn Baldwin 	if (os == NULL)
370b2e60773SJohn Baldwin 		return (ENOMEM);
371b2e60773SJohn Baldwin 
372b2e60773SJohn Baldwin 	crie.cri_alg = tls->params.cipher_algorithm;
373b2e60773SJohn Baldwin 	crie.cri_key = tls->params.cipher_key;
374b2e60773SJohn Baldwin 	crie.cri_klen = tls->params.cipher_key_len * 8;
375b2e60773SJohn Baldwin 
376b2e60773SJohn Baldwin 	crie.cri_next = &cria;
377b2e60773SJohn Baldwin 	error = crypto_newsession(&os->sid, &crie,
378b2e60773SJohn Baldwin 	    CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
379b2e60773SJohn Baldwin 	if (error) {
380b2e60773SJohn Baldwin 		free(os, M_KTLS_OCF);
381b2e60773SJohn Baldwin 		return (error);
382b2e60773SJohn Baldwin 	}
383b2e60773SJohn Baldwin 
384b2e60773SJohn Baldwin 	os->crda_alg = cria.cri_alg;
385b2e60773SJohn Baldwin 	mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF);
386b2e60773SJohn Baldwin 	tls->cipher = os;
387*55b7a0e1SJohn Baldwin 	if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)
388*55b7a0e1SJohn Baldwin 		tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt;
389*55b7a0e1SJohn Baldwin 	else
390*55b7a0e1SJohn Baldwin 		tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt;
391b2e60773SJohn Baldwin 	tls->free = ktls_ocf_free;
392b2e60773SJohn Baldwin 	return (0);
393b2e60773SJohn Baldwin }
394b2e60773SJohn Baldwin 
395b2e60773SJohn Baldwin struct ktls_crypto_backend ocf_backend = {
396b2e60773SJohn Baldwin 	.name = "OCF",
397b2e60773SJohn Baldwin 	.prio = 5,
398b2e60773SJohn Baldwin 	.api_version = KTLS_API_VERSION,
399b2e60773SJohn Baldwin 	.try = ktls_ocf_try,
400b2e60773SJohn Baldwin };
401b2e60773SJohn Baldwin 
402b2e60773SJohn Baldwin static int
403b2e60773SJohn Baldwin ktls_ocf_modevent(module_t mod, int what, void *arg)
404b2e60773SJohn Baldwin {
405b2e60773SJohn Baldwin 	int error;
406b2e60773SJohn Baldwin 
407b2e60773SJohn Baldwin 	switch (what) {
408b2e60773SJohn Baldwin 	case MOD_LOAD:
409*55b7a0e1SJohn Baldwin 		ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK);
410*55b7a0e1SJohn Baldwin 		ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK);
411b2e60773SJohn Baldwin 		ocf_retries = counter_u64_alloc(M_WAITOK);
412b2e60773SJohn Baldwin 		return (ktls_crypto_backend_register(&ocf_backend));
413b2e60773SJohn Baldwin 	case MOD_UNLOAD:
414b2e60773SJohn Baldwin 		error = ktls_crypto_backend_deregister(&ocf_backend);
415b2e60773SJohn Baldwin 		if (error)
416b2e60773SJohn Baldwin 			return (error);
417*55b7a0e1SJohn Baldwin 		counter_u64_free(ocf_tls12_gcm_crypts);
418*55b7a0e1SJohn Baldwin 		counter_u64_free(ocf_tls13_gcm_crypts);
419b2e60773SJohn Baldwin 		counter_u64_free(ocf_retries);
420b2e60773SJohn Baldwin 		return (0);
421b2e60773SJohn Baldwin 	default:
422b2e60773SJohn Baldwin 		return (EOPNOTSUPP);
423b2e60773SJohn Baldwin 	}
424b2e60773SJohn Baldwin }
425b2e60773SJohn Baldwin 
426b2e60773SJohn Baldwin static moduledata_t ktls_ocf_moduledata = {
427b2e60773SJohn Baldwin 	"ktls_ocf",
428b2e60773SJohn Baldwin 	ktls_ocf_modevent,
429b2e60773SJohn Baldwin 	NULL
430b2e60773SJohn Baldwin };
431b2e60773SJohn Baldwin 
432b2e60773SJohn Baldwin DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY);
433