xref: /freebsd/sys/crypto/blake2/blake2_cryptodev.c (revision cfd6422a5217410fbd66f7a7a8a64d9d85e61229)
1 /*-
2  * Copyright (c) 2018 Conrad Meyer <cem@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/bus.h>
32 #include <sys/kernel.h>
33 #include <sys/kobj.h>
34 #include <sys/lock.h>
35 #include <sys/malloc.h>
36 #include <sys/module.h>
37 #include <sys/mutex.h>
38 #include <sys/rwlock.h>
39 #include <sys/smp.h>
40 
41 #include <blake2.h>
42 
43 #include <opencrypto/cryptodev.h>
44 #include <cryptodev_if.h>
45 
46 #include <machine/fpu.h>
47 
48 struct blake2_session {
49 	size_t mlen;
50 };
51 CTASSERT((size_t)BLAKE2B_KEYBYTES > (size_t)BLAKE2S_KEYBYTES);
52 
53 struct blake2_softc {
54 	bool	dying;
55 	int32_t cid;
56 	struct rwlock lock;
57 };
58 
59 static struct mtx_padalign *ctx_mtx;
60 static struct fpu_kern_ctx **ctx_fpu;
61 
62 #define ACQUIRE_CTX(i, ctx)					\
63 	do {							\
64 		(i) = PCPU_GET(cpuid);				\
65 		mtx_lock(&ctx_mtx[(i)]);			\
66 		(ctx) = ctx_fpu[(i)];				\
67 	} while (0)
68 #define RELEASE_CTX(i, ctx)					\
69 	do {							\
70 		mtx_unlock(&ctx_mtx[(i)]);			\
71 		(i) = -1;					\
72 		(ctx) = NULL;					\
73 	} while (0)
74 
75 static int blake2_cipher_setup(struct blake2_session *ses,
76     const struct crypto_session_params *csp);
77 static int blake2_cipher_process(struct blake2_session *ses,
78     struct cryptop *crp);
79 
80 MALLOC_DEFINE(M_BLAKE2, "blake2_data", "Blake2 Data");
81 
82 static void
83 blake2_identify(driver_t *drv, device_t parent)
84 {
85 
86 	/* NB: order 10 is so we get attached after h/w devices */
87 	if (device_find_child(parent, "blaketwo", -1) == NULL &&
88 	    BUS_ADD_CHILD(parent, 10, "blaketwo", -1) == 0)
89 		panic("blaketwo: could not attach");
90 }
91 
92 static int
93 blake2_probe(device_t dev)
94 {
95 	device_set_desc(dev, "Blake2");
96 	return (0);
97 }
98 
99 static void
100 blake2_cleanctx(void)
101 {
102 	int i;
103 
104 	/* XXX - no way to return driverid */
105 	CPU_FOREACH(i) {
106 		if (ctx_fpu[i] != NULL) {
107 			mtx_destroy(&ctx_mtx[i]);
108 			fpu_kern_free_ctx(ctx_fpu[i]);
109 		}
110 		ctx_fpu[i] = NULL;
111 	}
112 	free(ctx_mtx, M_BLAKE2);
113 	ctx_mtx = NULL;
114 	free(ctx_fpu, M_BLAKE2);
115 	ctx_fpu = NULL;
116 }
117 
118 static int
119 blake2_attach(device_t dev)
120 {
121 	struct blake2_softc *sc;
122 	int i;
123 
124 	sc = device_get_softc(dev);
125 	sc->dying = false;
126 
127 	sc->cid = crypto_get_driverid(dev, sizeof(struct blake2_session),
128 	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
129 	    CRYPTOCAP_F_ACCEL_SOFTWARE);
130 	if (sc->cid < 0) {
131 		device_printf(dev, "Could not get crypto driver id.\n");
132 		return (ENOMEM);
133 	}
134 
135 	ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_BLAKE2,
136 	    M_WAITOK | M_ZERO);
137 	ctx_fpu = malloc(sizeof(*ctx_fpu) * (mp_maxid + 1), M_BLAKE2,
138 	    M_WAITOK | M_ZERO);
139 
140 	CPU_FOREACH(i) {
141 #ifdef __amd64__
142 		ctx_fpu[i] = fpu_kern_alloc_ctx_domain(
143 		    pcpu_find(i)->pc_domain, FPU_KERN_NORMAL);
144 #else
145 		ctx_fpu[i] = fpu_kern_alloc_ctx(FPU_KERN_NORMAL);
146 #endif
147 		mtx_init(&ctx_mtx[i], "bl2fpumtx", NULL, MTX_DEF | MTX_NEW);
148 	}
149 
150 	rw_init(&sc->lock, "blake2_lock");
151 
152 	return (0);
153 }
154 
155 static int
156 blake2_detach(device_t dev)
157 {
158 	struct blake2_softc *sc;
159 
160 	sc = device_get_softc(dev);
161 
162 	rw_wlock(&sc->lock);
163 	sc->dying = true;
164 	rw_wunlock(&sc->lock);
165 	crypto_unregister_all(sc->cid);
166 
167 	rw_destroy(&sc->lock);
168 
169 	blake2_cleanctx();
170 
171 	return (0);
172 }
173 
174 static int
175 blake2_probesession(device_t dev, const struct crypto_session_params *csp)
176 {
177 
178 	if (csp->csp_flags != 0)
179 		return (EINVAL);
180 	switch (csp->csp_mode) {
181 	case CSP_MODE_DIGEST:
182 		switch (csp->csp_auth_alg) {
183 		case CRYPTO_BLAKE2B:
184 		case CRYPTO_BLAKE2S:
185 			break;
186 		default:
187 			return (EINVAL);
188 		}
189 		break;
190 	default:
191 		return (EINVAL);
192 	}
193 	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
194 }
195 
196 static int
197 blake2_newsession(device_t dev, crypto_session_t cses,
198     const struct crypto_session_params *csp)
199 {
200 	struct blake2_softc *sc;
201 	struct blake2_session *ses;
202 	int error;
203 
204 	sc = device_get_softc(dev);
205 
206 	ses = crypto_get_driver_session(cses);
207 
208 	rw_rlock(&sc->lock);
209 	if (sc->dying) {
210 		rw_runlock(&sc->lock);
211 		return (EINVAL);
212 	}
213 	rw_runlock(&sc->lock);
214 
215 	error = blake2_cipher_setup(ses, csp);
216 	if (error != 0) {
217 		CRYPTDEB("setup failed");
218 		return (error);
219 	}
220 
221 	return (0);
222 }
223 
224 static int
225 blake2_process(device_t dev, struct cryptop *crp, int hint __unused)
226 {
227 	struct blake2_session *ses;
228 	int error;
229 
230 	ses = crypto_get_driver_session(crp->crp_session);
231 	error = blake2_cipher_process(ses, crp);
232 
233 	crp->crp_etype = error;
234 	crypto_done(crp);
235 	return (0);
236 }
237 
238 static device_method_t blake2_methods[] = {
239 	DEVMETHOD(device_identify, blake2_identify),
240 	DEVMETHOD(device_probe, blake2_probe),
241 	DEVMETHOD(device_attach, blake2_attach),
242 	DEVMETHOD(device_detach, blake2_detach),
243 
244 	DEVMETHOD(cryptodev_probesession, blake2_probesession),
245 	DEVMETHOD(cryptodev_newsession, blake2_newsession),
246 	DEVMETHOD(cryptodev_process, blake2_process),
247 
248 	DEVMETHOD_END
249 };
250 
251 static driver_t blake2_driver = {
252 	"blaketwo",
253 	blake2_methods,
254 	sizeof(struct blake2_softc),
255 };
256 static devclass_t blake2_devclass;
257 
258 DRIVER_MODULE(blake2, nexus, blake2_driver, blake2_devclass, 0, 0);
259 MODULE_VERSION(blake2, 1);
260 MODULE_DEPEND(blake2, crypto, 1, 1, 1);
261 
262 static bool
263 blake2_check_klen(const struct crypto_session_params *csp, unsigned klen)
264 {
265 
266 	if (csp->csp_auth_alg == CRYPTO_BLAKE2S)
267 		return (klen <= BLAKE2S_KEYBYTES);
268 	else
269 		return (klen <= BLAKE2B_KEYBYTES);
270 }
271 
272 static int
273 blake2_cipher_setup(struct blake2_session *ses,
274     const struct crypto_session_params *csp)
275 {
276 	int hashlen;
277 
278 	CTASSERT((size_t)BLAKE2S_OUTBYTES <= (size_t)BLAKE2B_OUTBYTES);
279 
280 	if (!blake2_check_klen(csp, csp->csp_auth_klen))
281 		return (EINVAL);
282 
283 	if (csp->csp_auth_mlen < 0)
284 		return (EINVAL);
285 
286 	switch (csp->csp_auth_alg) {
287 	case CRYPTO_BLAKE2S:
288 		hashlen = BLAKE2S_OUTBYTES;
289 		break;
290 	case CRYPTO_BLAKE2B:
291 		hashlen = BLAKE2B_OUTBYTES;
292 		break;
293 	default:
294 		return (EINVAL);
295 	}
296 
297 	if (csp->csp_auth_mlen > hashlen)
298 		return (EINVAL);
299 
300 	if (csp->csp_auth_mlen == 0)
301 		ses->mlen = hashlen;
302 	else
303 		ses->mlen = csp->csp_auth_mlen;
304 	return (0);
305 }
306 
307 static int
308 blake2b_applicator(void *state, const void *buf, u_int len)
309 {
310 	int rc;
311 
312 	rc = blake2b_update(state, buf, len);
313 	if (rc != 0)
314 		return (EINVAL);
315 	return (0);
316 }
317 
318 static int
319 blake2s_applicator(void *state, const void *buf, u_int len)
320 {
321 	int rc;
322 
323 	rc = blake2s_update(state, buf, len);
324 	if (rc != 0)
325 		return (EINVAL);
326 	return (0);
327 }
328 
329 static int
330 blake2_cipher_process(struct blake2_session *ses, struct cryptop *crp)
331 {
332 	union {
333 		blake2b_state sb;
334 		blake2s_state ss;
335 	} bctx;
336 	char res[BLAKE2B_OUTBYTES], res2[BLAKE2B_OUTBYTES];
337 	const struct crypto_session_params *csp;
338 	struct fpu_kern_ctx *ctx;
339 	const void *key;
340 	int ctxidx;
341 	bool kt;
342 	int error, rc;
343 	unsigned klen;
344 
345 	ctx = NULL;
346 	ctxidx = 0;
347 	error = EINVAL;
348 
349 	kt = is_fpu_kern_thread(0);
350 	if (!kt) {
351 		ACQUIRE_CTX(ctxidx, ctx);
352 		fpu_kern_enter(curthread, ctx,
353 		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
354 	}
355 
356 	csp = crypto_get_params(crp->crp_session);
357 	if (crp->crp_auth_key != NULL)
358 		key = crp->crp_auth_key;
359 	else
360 		key = csp->csp_auth_key;
361 	klen = csp->csp_auth_klen;
362 	switch (csp->csp_auth_alg) {
363 	case CRYPTO_BLAKE2B:
364 		if (klen > 0)
365 			rc = blake2b_init_key(&bctx.sb, ses->mlen, key, klen);
366 		else
367 			rc = blake2b_init(&bctx.sb, ses->mlen);
368 		if (rc != 0)
369 			goto out;
370 		error = crypto_apply(crp, crp->crp_payload_start,
371 		    crp->crp_payload_length, blake2b_applicator, &bctx.sb);
372 		if (error != 0)
373 			goto out;
374 		rc = blake2b_final(&bctx.sb, res, ses->mlen);
375 		if (rc != 0) {
376 			error = EINVAL;
377 			goto out;
378 		}
379 		break;
380 	case CRYPTO_BLAKE2S:
381 		if (klen > 0)
382 			rc = blake2s_init_key(&bctx.ss, ses->mlen, key, klen);
383 		else
384 			rc = blake2s_init(&bctx.ss, ses->mlen);
385 		if (rc != 0)
386 			goto out;
387 		error = crypto_apply(crp, crp->crp_payload_start,
388 		    crp->crp_payload_length, blake2s_applicator, &bctx.ss);
389 		if (error != 0)
390 			goto out;
391 		rc = blake2s_final(&bctx.ss, res, ses->mlen);
392 		if (rc != 0) {
393 			error = EINVAL;
394 			goto out;
395 		}
396 		break;
397 	default:
398 		panic("unreachable");
399 	}
400 
401 	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
402 		crypto_copydata(crp, crp->crp_digest_start, ses->mlen, res2);
403 		if (timingsafe_bcmp(res, res2, ses->mlen) != 0)
404 			return (EBADMSG);
405 	} else
406 		crypto_copyback(crp, crp->crp_digest_start, ses->mlen, res);
407 
408 out:
409 	if (!kt) {
410 		fpu_kern_leave(curthread, ctx);
411 		RELEASE_CTX(ctxidx, ctx);
412 	}
413 	return (error);
414 }
415