xref: /freebsd/sys/opencrypto/ktls_ocf.c (revision edca4938f74db18d091868237592abbf7e718669)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2019 Netflix Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/counter.h>
35 #include <sys/endian.h>
36 #include <sys/kernel.h>
37 #include <sys/ktls.h>
38 #include <sys/lock.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41 #include <sys/mutex.h>
42 #include <sys/sysctl.h>
43 #include <sys/uio.h>
44 #include <opencrypto/cryptodev.h>
45 
46 struct ocf_session {
47 	crypto_session_t sid;
48 	int crda_alg;
49 	struct mtx lock;
50 };
51 
52 struct ocf_operation {
53 	struct ocf_session *os;
54 	bool done;
55 	struct iovec iov[0];
56 };
57 
58 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS");
59 
60 SYSCTL_DECL(_kern_ipc_tls);
61 SYSCTL_DECL(_kern_ipc_tls_stats);
62 
63 static counter_u64_t ocf_gcm_crypts;
64 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, ocf_gcm_crypts, CTLFLAG_RD,
65     &ocf_gcm_crypts,
66     "Total number of OCF GCM encryption operations");
67 
68 static counter_u64_t ocf_retries;
69 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, ocf_retries, CTLFLAG_RD,
70     &ocf_retries,
71     "Number of OCF encryption operation retries");
72 
73 static int
74 ktls_ocf_callback(struct cryptop *crp)
75 {
76 	struct ocf_operation *oo;
77 
78 	oo = crp->crp_opaque;
79 	mtx_lock(&oo->os->lock);
80 	oo->done = true;
81 	mtx_unlock(&oo->os->lock);
82 	wakeup(oo);
83 	return (0);
84 }
85 
86 static int
87 ktls_ocf_encrypt(struct ktls_session *tls, const struct tls_record_layer *hdr,
88     uint8_t *trailer, struct iovec *iniov, struct iovec *outiov, int iovcnt,
89     uint64_t seqno, uint8_t record_type __unused)
90 {
91 	struct uio uio;
92 	struct tls_aead_data ad;
93 	struct tls_nonce_data nd;
94 	struct cryptodesc *crde, *crda;
95 	struct cryptop *crp;
96 	struct ocf_session *os;
97 	struct ocf_operation *oo;
98 	struct iovec *iov;
99 	int i, error;
100 	uint16_t tls_comp_len;
101 
102 	os = tls->cipher;
103 
104 	oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF,
105 	    M_WAITOK | M_ZERO);
106 	oo->os = os;
107 	iov = oo->iov;
108 
109 	crp = crypto_getreq(2);
110 	if (crp == NULL) {
111 		free(oo, M_KTLS_OCF);
112 		return (ENOMEM);
113 	}
114 
115 	/* Setup the IV. */
116 	memcpy(nd.fixed, tls->params.iv, TLS_AEAD_GCM_LEN);
117 	memcpy(&nd.seq, hdr + 1, sizeof(nd.seq));
118 
119 	/* Setup the AAD. */
120 	tls_comp_len = ntohs(hdr->tls_length) -
121 	    (AES_GMAC_HASH_LEN + sizeof(nd.seq));
122 	ad.seq = htobe64(seqno);
123 	ad.type = hdr->tls_type;
124 	ad.tls_vmajor = hdr->tls_vmajor;
125 	ad.tls_vminor = hdr->tls_vminor;
126 	ad.tls_length = htons(tls_comp_len);
127 	iov[0].iov_base = &ad;
128 	iov[0].iov_len = sizeof(ad);
129 	uio.uio_resid = sizeof(ad);
130 
131 	/*
132 	 * OCF always does encryption in place, so copy the data if
133 	 * needed.  Ugh.
134 	 */
135 	for (i = 0; i < iovcnt; i++) {
136 		iov[i + 1] = outiov[i];
137 		if (iniov[i].iov_base != outiov[i].iov_base)
138 			memcpy(outiov[i].iov_base, iniov[i].iov_base,
139 			    outiov[i].iov_len);
140 		uio.uio_resid += outiov[i].iov_len;
141 	}
142 
143 	iov[iovcnt + 1].iov_base = trailer;
144 	iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN;
145 	uio.uio_resid += AES_GMAC_HASH_LEN;
146 
147 	uio.uio_iov = iov;
148 	uio.uio_iovcnt = iovcnt + 2;
149 	uio.uio_offset = 0;
150 	uio.uio_segflg = UIO_SYSSPACE;
151 	uio.uio_td = curthread;
152 
153 	crp->crp_session = os->sid;
154 	crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM;
155 	crp->crp_uio = &uio;
156 	crp->crp_ilen = uio.uio_resid;
157 	crp->crp_opaque = oo;
158 	crp->crp_callback = ktls_ocf_callback;
159 
160 	crde = crp->crp_desc;
161 	crda = crde->crd_next;
162 
163 	crda->crd_alg = os->crda_alg;
164 	crda->crd_skip = 0;
165 	crda->crd_len = sizeof(ad);
166 	crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN;
167 
168 	crde->crd_alg = CRYPTO_AES_NIST_GCM_16;
169 	crde->crd_skip = sizeof(ad);
170 	crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN);
171 	crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
172 	memcpy(crde->crd_iv, &nd, sizeof(nd));
173 
174 	counter_u64_add(ocf_gcm_crypts, 1);
175 	for (;;) {
176 		error = crypto_dispatch(crp);
177 		if (error)
178 			break;
179 
180 		mtx_lock(&os->lock);
181 		while (!oo->done)
182 			mtx_sleep(oo, &os->lock, 0, "ocfktls", 0);
183 		mtx_unlock(&os->lock);
184 
185 		if (crp->crp_etype != EAGAIN) {
186 			error = crp->crp_etype;
187 			break;
188 		}
189 
190 		crp->crp_etype = 0;
191 		crp->crp_flags &= ~CRYPTO_F_DONE;
192 		oo->done = false;
193 		counter_u64_add(ocf_retries, 1);
194 	}
195 
196 	crypto_freereq(crp);
197 	free(oo, M_KTLS_OCF);
198 	return (error);
199 }
200 
201 static void
202 ktls_ocf_free(struct ktls_session *tls)
203 {
204 	struct ocf_session *os;
205 
206 	os = tls->cipher;
207 	mtx_destroy(&os->lock);
208 	explicit_bzero(os, sizeof(*os));
209 	free(os, M_KTLS_OCF);
210 }
211 
212 static int
213 ktls_ocf_try(struct socket *so, struct ktls_session *tls)
214 {
215 	struct cryptoini cria, crie;
216 	struct ocf_session *os;
217 	int error;
218 
219 	memset(&cria, 0, sizeof(cria));
220 	memset(&crie, 0, sizeof(crie));
221 
222 	switch (tls->params.cipher_algorithm) {
223 	case CRYPTO_AES_NIST_GCM_16:
224 		if (tls->params.iv_len != TLS_AEAD_GCM_LEN)
225 			return (EINVAL);
226 		switch (tls->params.cipher_key_len) {
227 		case 128 / 8:
228 			cria.cri_alg = CRYPTO_AES_128_NIST_GMAC;
229 			break;
230 		case 256 / 8:
231 			cria.cri_alg = CRYPTO_AES_256_NIST_GMAC;
232 			break;
233 		default:
234 			return (EINVAL);
235 		}
236 		cria.cri_key = tls->params.cipher_key;
237 		cria.cri_klen = tls->params.cipher_key_len * 8;
238 		break;
239 	default:
240 		return (EPROTONOSUPPORT);
241 	}
242 
243 	/* Only TLS 1.1 and TLS 1.2 are currently supported. */
244 	if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
245 	    tls->params.tls_vminor < TLS_MINOR_VER_ONE ||
246 	    tls->params.tls_vminor > TLS_MINOR_VER_TWO)
247 		return (EPROTONOSUPPORT);
248 
249 	os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO);
250 	if (os == NULL)
251 		return (ENOMEM);
252 
253 	crie.cri_alg = tls->params.cipher_algorithm;
254 	crie.cri_key = tls->params.cipher_key;
255 	crie.cri_klen = tls->params.cipher_key_len * 8;
256 
257 	crie.cri_next = &cria;
258 	error = crypto_newsession(&os->sid, &crie,
259 	    CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
260 	if (error) {
261 		free(os, M_KTLS_OCF);
262 		return (error);
263 	}
264 
265 	os->crda_alg = cria.cri_alg;
266 	mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF);
267 	tls->cipher = os;
268 	tls->sw_encrypt = ktls_ocf_encrypt;
269 	tls->free = ktls_ocf_free;
270 	return (0);
271 }
272 
273 struct ktls_crypto_backend ocf_backend = {
274 	.name = "OCF",
275 	.prio = 5,
276 	.api_version = KTLS_API_VERSION,
277 	.try = ktls_ocf_try,
278 };
279 
280 static int
281 ktls_ocf_modevent(module_t mod, int what, void *arg)
282 {
283 	int error;
284 
285 	switch (what) {
286 	case MOD_LOAD:
287 		ocf_gcm_crypts = counter_u64_alloc(M_WAITOK);
288 		ocf_retries = counter_u64_alloc(M_WAITOK);
289 		return (ktls_crypto_backend_register(&ocf_backend));
290 	case MOD_UNLOAD:
291 		error = ktls_crypto_backend_deregister(&ocf_backend);
292 		if (error)
293 			return (error);
294 		counter_u64_free(ocf_gcm_crypts);
295 		counter_u64_free(ocf_retries);
296 		return (0);
297 	default:
298 		return (EOPNOTSUPP);
299 	}
300 }
301 
302 static moduledata_t ktls_ocf_moduledata = {
303 	"ktls_ocf",
304 	ktls_ocf_modevent,
305 	NULL
306 };
307 
308 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY);
309