xref: /freebsd/sys/crypto/ccp/ccp.c (revision 0e2816f50ada7d863e26a630cdda327de449b1da) 
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2017 Chelsio Communications, Inc.
5  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
6  * All rights reserved.
7  * Largely borrowed from ccr(4), Written by: John Baldwin <jhb@FreeBSD.org>
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include "opt_ddb.h"
35 
36 #include <sys/param.h>
37 #include <sys/bus.h>
38 #include <sys/lock.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/mutex.h>
42 #include <sys/module.h>
43 #include <sys/random.h>
44 #include <sys/sglist.h>
45 #include <sys/sysctl.h>
46 
47 #ifdef DDB
48 #include <ddb/ddb.h>
49 #endif
50 
51 #include <dev/pci/pcivar.h>
52 
53 #include <dev/random/randomdev.h>
54 
55 #include <opencrypto/cryptodev.h>
56 #include <opencrypto/xform.h>
57 
58 #include "cryptodev_if.h"
59 
60 #include "ccp.h"
61 #include "ccp_hardware.h"
62 
63 MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto");
64 
65 /*
66  * Need a global softc available for garbage random_source API, which lacks any
67  * context pointer.  It's also handy for debugging.
68  */
69 struct ccp_softc *g_ccp_softc;
70 
71 bool g_debug_print = false;
72 SYSCTL_BOOL(_hw_ccp, OID_AUTO, debug, CTLFLAG_RWTUN, &g_debug_print, 0,
73     "Set to enable debugging log messages");
74 
75 static struct pciid {
76 	uint32_t devid;
77 	const char *desc;
78 } ccp_ids[] = {
79 	{ 0x14561022, "AMD CCP-5a" },
80 	{ 0x14681022, "AMD CCP-5b" },
81 };
82 
83 static struct random_source random_ccp = {
84 	.rs_ident = "AMD CCP TRNG",
85 	.rs_source = RANDOM_PURE_CCP,
86 	.rs_read = random_ccp_read,
87 };
88 
89 /*
90  * ccp_populate_sglist() generates a scatter/gather list that covers the entire
91  * crypto operation buffer.
92  */
93 static int
94 ccp_populate_sglist(struct sglist *sg, struct cryptop *crp)
95 {
96 	int error;
97 
98 	sglist_reset(sg);
99 	switch (crp->crp_buf_type) {
100 	case CRYPTO_BUF_MBUF:
101 		error = sglist_append_mbuf(sg, crp->crp_mbuf);
102 		break;
103 	case CRYPTO_BUF_UIO:
104 		error = sglist_append_uio(sg, crp->crp_uio);
105 		break;
106 	case CRYPTO_BUF_CONTIG:
107 		error = sglist_append(sg, crp->crp_buf, crp->crp_ilen);
108 		break;
109 	default:
110 		error = EINVAL;
111 	}
112 	return (error);
113 }
114 
115 /*
116  * Handle a GCM request with an empty payload by performing the
117  * operation in software.
118  */
119 static void
120 ccp_gcm_soft(struct ccp_session *s, struct cryptop *crp)
121 {
122 	struct aes_gmac_ctx gmac_ctx;
123 	char block[GMAC_BLOCK_LEN];
124 	char digest[GMAC_DIGEST_LEN];
125 	char iv[AES_BLOCK_LEN];
126 	int i, len;
127 
128 	/*
129 	 * This assumes a 12-byte IV from the crp.  See longer comment
130 	 * above in ccp_gcm() for more details.
131 	 */
132 	if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) {
133 		crp->crp_etype = EINVAL;
134 		goto out;
135 	}
136 	memcpy(iv, crp->crp_iv, 12);
137 	*(uint32_t *)&iv[12] = htobe32(1);
138 
139 	/* Initialize the MAC. */
140 	AES_GMAC_Init(&gmac_ctx);
141 	AES_GMAC_Setkey(&gmac_ctx, s->blkcipher.enckey, s->blkcipher.key_len);
142 	AES_GMAC_Reinit(&gmac_ctx, iv, sizeof(iv));
143 
144 	/* MAC the AAD. */
145 	for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) {
146 		len = imin(crp->crp_aad_length - i, sizeof(block));
147 		crypto_copydata(crp, crp->crp_aad_start + i, len, block);
148 		bzero(block + len, sizeof(block) - len);
149 		AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
150 	}
151 
152 	/* Length block. */
153 	bzero(block, sizeof(block));
154 	((uint32_t *)block)[1] = htobe32(crp->crp_aad_length * 8);
155 	AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
156 	AES_GMAC_Final(digest, &gmac_ctx);
157 
158 	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
159 		crypto_copyback(crp, crp->crp_digest_start, sizeof(digest),
160 		    digest);
161 		crp->crp_etype = 0;
162 	} else {
163 		char digest2[GMAC_DIGEST_LEN];
164 
165 		crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2),
166 		    digest2);
167 		if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0)
168 			crp->crp_etype = 0;
169 		else
170 			crp->crp_etype = EBADMSG;
171 	}
172 out:
173 	crypto_done(crp);
174 }
175 
176 static int
177 ccp_probe(device_t dev)
178 {
179 	struct pciid *ip;
180 	uint32_t id;
181 
182 	id = pci_get_devid(dev);
183 	for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) {
184 		if (id == ip->devid) {
185 			device_set_desc(dev, ip->desc);
186 			return (0);
187 		}
188 	}
189 	return (ENXIO);
190 }
191 
192 static void
193 ccp_initialize_queues(struct ccp_softc *sc)
194 {
195 	struct ccp_queue *qp;
196 	size_t i;
197 
198 	for (i = 0; i < nitems(sc->queues); i++) {
199 		qp = &sc->queues[i];
200 
201 		qp->cq_softc = sc;
202 		qp->cq_qindex = i;
203 		mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF);
204 		/* XXX - arbitrarily chosen sizes */
205 		qp->cq_sg_crp = sglist_alloc(32, M_WAITOK);
206 		/* Two more SGEs than sg_crp to accommodate ipad. */
207 		qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK);
208 		qp->cq_sg_dst = sglist_alloc(2, M_WAITOK);
209 	}
210 }
211 
212 static void
213 ccp_free_queues(struct ccp_softc *sc)
214 {
215 	struct ccp_queue *qp;
216 	size_t i;
217 
218 	for (i = 0; i < nitems(sc->queues); i++) {
219 		qp = &sc->queues[i];
220 
221 		mtx_destroy(&qp->cq_lock);
222 		sglist_free(qp->cq_sg_crp);
223 		sglist_free(qp->cq_sg_ulptx);
224 		sglist_free(qp->cq_sg_dst);
225 	}
226 }
227 
228 static int
229 ccp_attach(device_t dev)
230 {
231 	struct ccp_softc *sc;
232 	int error;
233 
234 	sc = device_get_softc(dev);
235 	sc->dev = dev;
236 
237 	sc->cid = crypto_get_driverid(dev, sizeof(struct ccp_session),
238 	    CRYPTOCAP_F_HARDWARE);
239 	if (sc->cid < 0) {
240 		device_printf(dev, "could not get crypto driver id\n");
241 		return (ENXIO);
242 	}
243 
244 	error = ccp_hw_attach(dev);
245 	if (error != 0)
246 		return (error);
247 
248 	mtx_init(&sc->lock, "ccp", NULL, MTX_DEF);
249 
250 	ccp_initialize_queues(sc);
251 
252 	if (g_ccp_softc == NULL) {
253 		g_ccp_softc = sc;
254 		if ((sc->hw_features & VERSION_CAP_TRNG) != 0)
255 			random_source_register(&random_ccp);
256 	}
257 
258 	return (0);
259 }
260 
261 static int
262 ccp_detach(device_t dev)
263 {
264 	struct ccp_softc *sc;
265 
266 	sc = device_get_softc(dev);
267 
268 	mtx_lock(&sc->lock);
269 	sc->detaching = true;
270 	mtx_unlock(&sc->lock);
271 
272 	crypto_unregister_all(sc->cid);
273 	if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0)
274 		random_source_deregister(&random_ccp);
275 
276 	ccp_hw_detach(dev);
277 	ccp_free_queues(sc);
278 
279 	if (g_ccp_softc == sc)
280 		g_ccp_softc = NULL;
281 
282 	mtx_destroy(&sc->lock);
283 	return (0);
284 }
285 
286 static void
287 ccp_init_hmac_digest(struct ccp_session *s, const char *key, int klen)
288 {
289 	union authctx auth_ctx;
290 	struct auth_hash *axf;
291 	u_int i;
292 
293 	/*
294 	 * If the key is larger than the block size, use the digest of
295 	 * the key as the key instead.
296 	 */
297 	axf = s->hmac.auth_hash;
298 	if (klen > axf->blocksize) {
299 		axf->Init(&auth_ctx);
300 		axf->Update(&auth_ctx, key, klen);
301 		axf->Final(s->hmac.ipad, &auth_ctx);
302 		explicit_bzero(&auth_ctx, sizeof(auth_ctx));
303 		klen = axf->hashsize;
304 	} else
305 		memcpy(s->hmac.ipad, key, klen);
306 
307 	memset(s->hmac.ipad + klen, 0, axf->blocksize - klen);
308 	memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize);
309 
310 	for (i = 0; i < axf->blocksize; i++) {
311 		s->hmac.ipad[i] ^= HMAC_IPAD_VAL;
312 		s->hmac.opad[i] ^= HMAC_OPAD_VAL;
313 	}
314 }
315 
316 static bool
317 ccp_aes_check_keylen(int alg, int klen)
318 {
319 
320 	switch (klen * 8) {
321 	case 128:
322 	case 192:
323 		if (alg == CRYPTO_AES_XTS)
324 			return (false);
325 		break;
326 	case 256:
327 		break;
328 	case 512:
329 		if (alg != CRYPTO_AES_XTS)
330 			return (false);
331 		break;
332 	default:
333 		return (false);
334 	}
335 	return (true);
336 }
337 
338 static void
339 ccp_aes_setkey(struct ccp_session *s, int alg, const void *key, int klen)
340 {
341 	unsigned kbits;
342 
343 	if (alg == CRYPTO_AES_XTS)
344 		kbits = (klen / 2) * 8;
345 	else
346 		kbits = klen * 8;
347 
348 	switch (kbits) {
349 	case 128:
350 		s->blkcipher.cipher_type = CCP_AES_TYPE_128;
351 		break;
352 	case 192:
353 		s->blkcipher.cipher_type = CCP_AES_TYPE_192;
354 		break;
355 	case 256:
356 		s->blkcipher.cipher_type = CCP_AES_TYPE_256;
357 		break;
358 	default:
359 		panic("should not get here");
360 	}
361 
362 	s->blkcipher.key_len = klen;
363 	memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
364 }
365 
366 static bool
367 ccp_auth_supported(struct ccp_softc *sc,
368     const struct crypto_session_params *csp)
369 {
370 
371 	if ((sc->hw_features & VERSION_CAP_SHA) == 0)
372 		return (false);
373 	switch (csp->csp_auth_alg) {
374 	case CRYPTO_SHA1_HMAC:
375 	case CRYPTO_SHA2_256_HMAC:
376 	case CRYPTO_SHA2_384_HMAC:
377 	case CRYPTO_SHA2_512_HMAC:
378 		if (csp->csp_auth_key == NULL)
379 			return (false);
380 		break;
381 	default:
382 		return (false);
383 	}
384 	return (true);
385 }
386 
387 static bool
388 ccp_cipher_supported(struct ccp_softc *sc,
389     const struct crypto_session_params *csp)
390 {
391 
392 	if ((sc->hw_features & VERSION_CAP_AES) == 0)
393 		return (false);
394 	switch (csp->csp_cipher_alg) {
395 	case CRYPTO_AES_CBC:
396 		if (csp->csp_ivlen != AES_BLOCK_LEN)
397 			return (false);
398 		break;
399 	case CRYPTO_AES_ICM:
400 		if (csp->csp_ivlen != AES_BLOCK_LEN)
401 			return (false);
402 		break;
403 	case CRYPTO_AES_XTS:
404 		if (csp->csp_ivlen != AES_XTS_IV_LEN)
405 			return (false);
406 		break;
407 	default:
408 		return (false);
409 	}
410 	return (ccp_aes_check_keylen(csp->csp_cipher_alg,
411 	    csp->csp_cipher_klen));
412 }
413 
414 static int
415 ccp_probesession(device_t dev, const struct crypto_session_params *csp)
416 {
417 	struct ccp_softc *sc;
418 
419 	if (csp->csp_flags != 0)
420 		return (EINVAL);
421 	sc = device_get_softc(dev);
422 	switch (csp->csp_mode) {
423 	case CSP_MODE_DIGEST:
424 		if (!ccp_auth_supported(sc, csp))
425 			return (EINVAL);
426 		break;
427 	case CSP_MODE_CIPHER:
428 		if (!ccp_cipher_supported(sc, csp))
429 			return (EINVAL);
430 		break;
431 	case CSP_MODE_AEAD:
432 		switch (csp->csp_cipher_alg) {
433 		case CRYPTO_AES_NIST_GCM_16:
434 			if (csp->csp_ivlen != AES_GCM_IV_LEN)
435 				return (EINVAL);
436 			if (csp->csp_auth_mlen < 0 ||
437 			    csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
438 				return (EINVAL);
439 			if ((sc->hw_features & VERSION_CAP_AES) == 0)
440 				return (EINVAL);
441 			break;
442 		default:
443 			return (EINVAL);
444 		}
445 		break;
446 	case CSP_MODE_ETA:
447 		if (!ccp_auth_supported(sc, csp) ||
448 		    !ccp_cipher_supported(sc, csp))
449 			return (EINVAL);
450 		break;
451 	default:
452 		return (EINVAL);
453 	}
454 
455 	return (CRYPTODEV_PROBE_HARDWARE);
456 }
457 
458 static int
459 ccp_newsession(device_t dev, crypto_session_t cses,
460     const struct crypto_session_params *csp)
461 {
462 	struct ccp_softc *sc;
463 	struct ccp_session *s;
464 	struct auth_hash *auth_hash;
465 	enum ccp_aes_mode cipher_mode;
466 	unsigned auth_mode;
467 	unsigned q;
468 
469 	/* XXX reconcile auth_mode with use by ccp_sha */
470 	switch (csp->csp_auth_alg) {
471 	case CRYPTO_SHA1_HMAC:
472 		auth_hash = &auth_hash_hmac_sha1;
473 		auth_mode = SHA1;
474 		break;
475 	case CRYPTO_SHA2_256_HMAC:
476 		auth_hash = &auth_hash_hmac_sha2_256;
477 		auth_mode = SHA2_256;
478 		break;
479 	case CRYPTO_SHA2_384_HMAC:
480 		auth_hash = &auth_hash_hmac_sha2_384;
481 		auth_mode = SHA2_384;
482 		break;
483 	case CRYPTO_SHA2_512_HMAC:
484 		auth_hash = &auth_hash_hmac_sha2_512;
485 		auth_mode = SHA2_512;
486 		break;
487 	default:
488 		auth_hash = NULL;
489 		auth_mode = 0;
490 		break;
491 	}
492 
493 	switch (csp->csp_cipher_alg) {
494 	case CRYPTO_AES_CBC:
495 		cipher_mode = CCP_AES_MODE_CBC;
496 		break;
497 	case CRYPTO_AES_ICM:
498 		cipher_mode = CCP_AES_MODE_CTR;
499 		break;
500 	case CRYPTO_AES_NIST_GCM_16:
501 		cipher_mode = CCP_AES_MODE_GCTR;
502 		break;
503 	case CRYPTO_AES_XTS:
504 		cipher_mode = CCP_AES_MODE_XTS;
505 		break;
506 	default:
507 		cipher_mode = CCP_AES_MODE_ECB;
508 		break;
509 	}
510 
511 	sc = device_get_softc(dev);
512 	mtx_lock(&sc->lock);
513 	if (sc->detaching) {
514 		mtx_unlock(&sc->lock);
515 		return (ENXIO);
516 	}
517 
518 	s = crypto_get_driver_session(cses);
519 
520 	/* Just grab the first usable queue for now. */
521 	for (q = 0; q < nitems(sc->queues); q++)
522 		if ((sc->valid_queues & (1 << q)) != 0)
523 			break;
524 	if (q == nitems(sc->queues)) {
525 		mtx_unlock(&sc->lock);
526 		return (ENXIO);
527 	}
528 	s->queue = q;
529 
530 	switch (csp->csp_mode) {
531 	case CSP_MODE_AEAD:
532 		s->mode = GCM;
533 		break;
534 	case CSP_MODE_ETA:
535 		s->mode = AUTHENC;
536 		break;
537 	case CSP_MODE_DIGEST:
538 		s->mode = HMAC;
539 		break;
540 	case CSP_MODE_CIPHER:
541 		s->mode = BLKCIPHER;
542 		break;
543 	}
544 
545 	if (s->mode == GCM) {
546 		if (csp->csp_auth_mlen == 0)
547 			s->gmac.hash_len = AES_GMAC_HASH_LEN;
548 		else
549 			s->gmac.hash_len = csp->csp_auth_mlen;
550 	} else if (auth_hash != NULL) {
551 		s->hmac.auth_hash = auth_hash;
552 		s->hmac.auth_mode = auth_mode;
553 		if (csp->csp_auth_mlen == 0)
554 			s->hmac.hash_len = auth_hash->hashsize;
555 		else
556 			s->hmac.hash_len = csp->csp_auth_mlen;
557 		ccp_init_hmac_digest(s, csp->csp_auth_key, csp->csp_auth_klen);
558 	}
559 	if (cipher_mode != CCP_AES_MODE_ECB) {
560 		s->blkcipher.cipher_mode = cipher_mode;
561 		if (csp->csp_cipher_key != NULL)
562 			ccp_aes_setkey(s, csp->csp_cipher_alg,
563 			    csp->csp_cipher_key, csp->csp_cipher_klen);
564 	}
565 
566 	s->active = true;
567 	mtx_unlock(&sc->lock);
568 
569 	return (0);
570 }
571 
572 static void
573 ccp_freesession(device_t dev, crypto_session_t cses)
574 {
575 	struct ccp_session *s;
576 
577 	s = crypto_get_driver_session(cses);
578 
579 	if (s->pending != 0)
580 		device_printf(dev,
581 		    "session %p freed with %d pending requests\n", s,
582 		    s->pending);
583 	s->active = false;
584 }
585 
586 static int
587 ccp_process(device_t dev, struct cryptop *crp, int hint)
588 {
589 	const struct crypto_session_params *csp;
590 	struct ccp_softc *sc;
591 	struct ccp_queue *qp;
592 	struct ccp_session *s;
593 	int error;
594 	bool qpheld;
595 
596 	qpheld = false;
597 	qp = NULL;
598 
599 	csp = crypto_get_params(crp->crp_session);
600 	s = crypto_get_driver_session(crp->crp_session);
601 	sc = device_get_softc(dev);
602 	mtx_lock(&sc->lock);
603 	qp = &sc->queues[s->queue];
604 	mtx_unlock(&sc->lock);
605 	error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT);
606 	if (error != 0)
607 		goto out;
608 	qpheld = true;
609 
610 	error = ccp_populate_sglist(qp->cq_sg_crp, crp);
611 	if (error != 0)
612 		goto out;
613 
614 	if (crp->crp_auth_key != NULL) {
615 		KASSERT(s->hmac.auth_hash != NULL, ("auth key without HMAC"));
616 		ccp_init_hmac_digest(s, crp->crp_auth_key, csp->csp_auth_klen);
617 	}
618 	if (crp->crp_cipher_key != NULL)
619 		ccp_aes_setkey(s, csp->csp_cipher_alg, crp->crp_cipher_key,
620 		    csp->csp_cipher_klen);
621 
622 	switch (s->mode) {
623 	case HMAC:
624 		if (s->pending != 0) {
625 			error = EAGAIN;
626 			break;
627 		}
628 		error = ccp_hmac(qp, s, crp);
629 		break;
630 	case BLKCIPHER:
631 		if (s->pending != 0) {
632 			error = EAGAIN;
633 			break;
634 		}
635 		error = ccp_blkcipher(qp, s, crp);
636 		break;
637 	case AUTHENC:
638 		if (s->pending != 0) {
639 			error = EAGAIN;
640 			break;
641 		}
642 		error = ccp_authenc(qp, s, crp);
643 		break;
644 	case GCM:
645 		if (crp->crp_payload_length == 0) {
646 			mtx_unlock(&qp->cq_lock);
647 			ccp_gcm_soft(s, crp);
648 			return (0);
649 		}
650 		if (s->pending != 0) {
651 			error = EAGAIN;
652 			break;
653 		}
654 		error = ccp_gcm(qp, s, crp);
655 		break;
656 	}
657 
658 	if (error == 0)
659 		s->pending++;
660 
661 out:
662 	if (qpheld) {
663 		if (error != 0) {
664 			/*
665 			 * Squash EAGAIN so callers don't uselessly and
666 			 * expensively retry if the ring was full.
667 			 */
668 			if (error == EAGAIN)
669 				error = ENOMEM;
670 			ccp_queue_abort(qp);
671 		} else
672 			ccp_queue_release(qp);
673 	}
674 
675 	if (error != 0) {
676 		DPRINTF(dev, "%s: early error:%d\n", __func__, error);
677 		crp->crp_etype = error;
678 		crypto_done(crp);
679 	}
680 	return (0);
681 }
682 
683 static device_method_t ccp_methods[] = {
684 	DEVMETHOD(device_probe,		ccp_probe),
685 	DEVMETHOD(device_attach,	ccp_attach),
686 	DEVMETHOD(device_detach,	ccp_detach),
687 
688 	DEVMETHOD(cryptodev_probesession, ccp_probesession),
689 	DEVMETHOD(cryptodev_newsession,	ccp_newsession),
690 	DEVMETHOD(cryptodev_freesession, ccp_freesession),
691 	DEVMETHOD(cryptodev_process,	ccp_process),
692 
693 	DEVMETHOD_END
694 };
695 
696 static driver_t ccp_driver = {
697 	"ccp",
698 	ccp_methods,
699 	sizeof(struct ccp_softc)
700 };
701 
702 static devclass_t ccp_devclass;
703 DRIVER_MODULE(ccp, pci, ccp_driver, ccp_devclass, NULL, NULL);
704 MODULE_VERSION(ccp, 1);
705 MODULE_DEPEND(ccp, crypto, 1, 1, 1);
706 MODULE_DEPEND(ccp, random_device, 1, 1, 1);
707 #if 0	/* There are enough known issues that we shouldn't load automatically */
708 MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids,
709     nitems(ccp_ids));
710 #endif
711 
712 static int
713 ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags)
714 {
715 	struct ccp_softc *sc;
716 
717 	mtx_assert(&qp->cq_lock, MA_OWNED);
718 	sc = qp->cq_softc;
719 
720 	if (n < 1 || n >= (1 << sc->ring_size_order))
721 		return (EINVAL);
722 
723 	while (true) {
724 		if (ccp_queue_get_ring_space(qp) >= n)
725 			return (0);
726 		if ((mflags & M_WAITOK) == 0)
727 			return (EAGAIN);
728 		qp->cq_waiting = true;
729 		msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0);
730 	}
731 }
732 
733 int
734 ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags)
735 {
736 	int error;
737 
738 	mtx_lock(&qp->cq_lock);
739 	qp->cq_acq_tail = qp->cq_tail;
740 	error = ccp_queue_reserve_space(qp, n, mflags);
741 	if (error != 0)
742 		mtx_unlock(&qp->cq_lock);
743 	return (error);
744 }
745 
746 void
747 ccp_queue_release(struct ccp_queue *qp)
748 {
749 
750 	mtx_assert(&qp->cq_lock, MA_OWNED);
751 	if (qp->cq_tail != qp->cq_acq_tail) {
752 		wmb();
753 		ccp_queue_write_tail(qp);
754 	}
755 	mtx_unlock(&qp->cq_lock);
756 }
757 
758 void
759 ccp_queue_abort(struct ccp_queue *qp)
760 {
761 	unsigned i;
762 
763 	mtx_assert(&qp->cq_lock, MA_OWNED);
764 
765 	/* Wipe out any descriptors associated with this aborted txn. */
766 	for (i = qp->cq_acq_tail; i != qp->cq_tail;
767 	    i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) {
768 		memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i]));
769 	}
770 	qp->cq_tail = qp->cq_acq_tail;
771 
772 	mtx_unlock(&qp->cq_lock);
773 }
774 
775 #ifdef DDB
776 #define	_db_show_lock(lo)	LOCK_CLASS(lo)->lc_ddb_show(lo)
777 #define	db_show_lock(lk)	_db_show_lock(&(lk)->lock_object)
778 static void
779 db_show_ccp_sc(struct ccp_softc *sc)
780 {
781 
782 	db_printf("ccp softc at %p\n", sc);
783 	db_printf(" cid: %d\n", (int)sc->cid);
784 
785 	db_printf(" lock: ");
786 	db_show_lock(&sc->lock);
787 
788 	db_printf(" detaching: %d\n", (int)sc->detaching);
789 	db_printf(" ring_size_order: %u\n", sc->ring_size_order);
790 
791 	db_printf(" hw_version: %d\n", (int)sc->hw_version);
792 	db_printf(" hw_features: %b\n", (int)sc->hw_features,
793 	    "\20\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA"
794 	    "\11SHA\0103DES\07AES");
795 
796 	db_printf(" hw status:\n");
797 	db_ccp_show_hw(sc);
798 }
799 
800 static void
801 db_show_ccp_qp(struct ccp_queue *qp)
802 {
803 
804 	db_printf(" lock: ");
805 	db_show_lock(&qp->cq_lock);
806 
807 	db_printf(" cq_qindex: %u\n", qp->cq_qindex);
808 	db_printf(" cq_softc: %p\n", qp->cq_softc);
809 
810 	db_printf(" head: %u\n", qp->cq_head);
811 	db_printf(" tail: %u\n", qp->cq_tail);
812 	db_printf(" acq_tail: %u\n", qp->cq_acq_tail);
813 	db_printf(" desc_ring: %p\n", qp->desc_ring);
814 	db_printf(" completions_ring: %p\n", qp->completions_ring);
815 	db_printf(" descriptors (phys): 0x%jx\n",
816 	    (uintmax_t)qp->desc_ring_bus_addr);
817 
818 	db_printf(" hw status:\n");
819 	db_ccp_show_queue_hw(qp);
820 }
821 
822 DB_SHOW_COMMAND(ccp, db_show_ccp)
823 {
824 	struct ccp_softc *sc;
825 	unsigned unit, qindex;
826 
827 	if (!have_addr)
828 		goto usage;
829 
830 	unit = (unsigned)addr;
831 
832 	sc = devclass_get_softc(ccp_devclass, unit);
833 	if (sc == NULL) {
834 		db_printf("No such device ccp%u\n", unit);
835 		goto usage;
836 	}
837 
838 	if (count == -1) {
839 		db_show_ccp_sc(sc);
840 		return;
841 	}
842 
843 	qindex = (unsigned)count;
844 	if (qindex >= nitems(sc->queues)) {
845 		db_printf("No such queue %u\n", qindex);
846 		goto usage;
847 	}
848 	db_show_ccp_qp(&sc->queues[qindex]);
849 	return;
850 
851 usage:
852 	db_printf("usage: show ccp <unit>[,<qindex>]\n");
853 	return;
854 }
855 #endif /* DDB */
856