xref: /freebsd/sys/crypto/ccp/ccp.c (revision 060a805b2f0aaa503e8fc2729e07c657d5ee24b2)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2017 Chelsio Communications, Inc.
5  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
6  * All rights reserved.
7  * Largely borrowed from ccr(4), Written by: John Baldwin <jhb@FreeBSD.org>
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include "opt_ddb.h"
35 
36 #include <sys/param.h>
37 #include <sys/bus.h>
38 #include <sys/lock.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/mutex.h>
42 #include <sys/module.h>
43 #include <sys/random.h>
44 #include <sys/sglist.h>
45 #include <sys/sysctl.h>
46 
47 #ifdef DDB
48 #include <ddb/ddb.h>
49 #endif
50 
51 #include <dev/pci/pcivar.h>
52 
53 #include <dev/random/randomdev.h>
54 
55 #include <opencrypto/cryptodev.h>
56 #include <opencrypto/xform.h>
57 
58 #include "cryptodev_if.h"
59 
60 #include "ccp.h"
61 #include "ccp_hardware.h"
62 
63 MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto");
64 
65 /*
66  * Need a global softc available for garbage random_source API, which lacks any
67  * context pointer.  It's also handy for debugging.
68  */
69 struct ccp_softc *g_ccp_softc;
70 
71 bool g_debug_print = false;
72 SYSCTL_BOOL(_hw_ccp, OID_AUTO, debug, CTLFLAG_RWTUN, &g_debug_print, 0,
73     "Set to enable debugging log messages");
74 
75 static struct pciid {
76 	uint32_t devid;
77 	const char *desc;
78 } ccp_ids[] = {
79 	{ 0x14561022, "AMD CCP-5a" },
80 	{ 0x14681022, "AMD CCP-5b" },
81 	{ 0x15df1022, "AMD CCP-5a" },
82 };
83 
84 static struct random_source random_ccp = {
85 	.rs_ident = "AMD CCP TRNG",
86 	.rs_source = RANDOM_PURE_CCP,
87 	.rs_read = random_ccp_read,
88 };
89 
90 /*
91  * ccp_populate_sglist() generates a scatter/gather list that covers the entire
92  * crypto operation buffer.
93  */
94 static int
95 ccp_populate_sglist(struct sglist *sg, struct cryptop *crp)
96 {
97 	int error;
98 
99 	sglist_reset(sg);
100 	switch (crp->crp_buf_type) {
101 	case CRYPTO_BUF_MBUF:
102 		error = sglist_append_mbuf(sg, crp->crp_mbuf);
103 		break;
104 	case CRYPTO_BUF_UIO:
105 		error = sglist_append_uio(sg, crp->crp_uio);
106 		break;
107 	case CRYPTO_BUF_CONTIG:
108 		error = sglist_append(sg, crp->crp_buf, crp->crp_ilen);
109 		break;
110 	default:
111 		error = EINVAL;
112 	}
113 	return (error);
114 }
115 
116 /*
117  * Handle a GCM request with an empty payload by performing the
118  * operation in software.
119  */
120 static void
121 ccp_gcm_soft(struct ccp_session *s, struct cryptop *crp)
122 {
123 	struct aes_gmac_ctx gmac_ctx;
124 	char block[GMAC_BLOCK_LEN];
125 	char digest[GMAC_DIGEST_LEN];
126 	char iv[AES_BLOCK_LEN];
127 	int i, len;
128 
129 	/*
130 	 * This assumes a 12-byte IV from the crp.  See longer comment
131 	 * above in ccp_gcm() for more details.
132 	 */
133 	if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) {
134 		crp->crp_etype = EINVAL;
135 		goto out;
136 	}
137 	memcpy(iv, crp->crp_iv, 12);
138 	*(uint32_t *)&iv[12] = htobe32(1);
139 
140 	/* Initialize the MAC. */
141 	AES_GMAC_Init(&gmac_ctx);
142 	AES_GMAC_Setkey(&gmac_ctx, s->blkcipher.enckey, s->blkcipher.key_len);
143 	AES_GMAC_Reinit(&gmac_ctx, iv, sizeof(iv));
144 
145 	/* MAC the AAD. */
146 	for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) {
147 		len = imin(crp->crp_aad_length - i, sizeof(block));
148 		crypto_copydata(crp, crp->crp_aad_start + i, len, block);
149 		bzero(block + len, sizeof(block) - len);
150 		AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
151 	}
152 
153 	/* Length block. */
154 	bzero(block, sizeof(block));
155 	((uint32_t *)block)[1] = htobe32(crp->crp_aad_length * 8);
156 	AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
157 	AES_GMAC_Final(digest, &gmac_ctx);
158 
159 	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
160 		crypto_copyback(crp, crp->crp_digest_start, sizeof(digest),
161 		    digest);
162 		crp->crp_etype = 0;
163 	} else {
164 		char digest2[GMAC_DIGEST_LEN];
165 
166 		crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2),
167 		    digest2);
168 		if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0)
169 			crp->crp_etype = 0;
170 		else
171 			crp->crp_etype = EBADMSG;
172 	}
173 out:
174 	crypto_done(crp);
175 }
176 
177 static int
178 ccp_probe(device_t dev)
179 {
180 	struct pciid *ip;
181 	uint32_t id;
182 
183 	id = pci_get_devid(dev);
184 	for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) {
185 		if (id == ip->devid) {
186 			device_set_desc(dev, ip->desc);
187 			return (0);
188 		}
189 	}
190 	return (ENXIO);
191 }
192 
193 static void
194 ccp_initialize_queues(struct ccp_softc *sc)
195 {
196 	struct ccp_queue *qp;
197 	size_t i;
198 
199 	for (i = 0; i < nitems(sc->queues); i++) {
200 		qp = &sc->queues[i];
201 
202 		qp->cq_softc = sc;
203 		qp->cq_qindex = i;
204 		mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF);
205 		/* XXX - arbitrarily chosen sizes */
206 		qp->cq_sg_crp = sglist_alloc(32, M_WAITOK);
207 		/* Two more SGEs than sg_crp to accommodate ipad. */
208 		qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK);
209 		qp->cq_sg_dst = sglist_alloc(2, M_WAITOK);
210 	}
211 }
212 
213 static void
214 ccp_free_queues(struct ccp_softc *sc)
215 {
216 	struct ccp_queue *qp;
217 	size_t i;
218 
219 	for (i = 0; i < nitems(sc->queues); i++) {
220 		qp = &sc->queues[i];
221 
222 		mtx_destroy(&qp->cq_lock);
223 		sglist_free(qp->cq_sg_crp);
224 		sglist_free(qp->cq_sg_ulptx);
225 		sglist_free(qp->cq_sg_dst);
226 	}
227 }
228 
229 static int
230 ccp_attach(device_t dev)
231 {
232 	struct ccp_softc *sc;
233 	int error;
234 
235 	sc = device_get_softc(dev);
236 	sc->dev = dev;
237 
238 	sc->cid = crypto_get_driverid(dev, sizeof(struct ccp_session),
239 	    CRYPTOCAP_F_HARDWARE);
240 	if (sc->cid < 0) {
241 		device_printf(dev, "could not get crypto driver id\n");
242 		return (ENXIO);
243 	}
244 
245 	error = ccp_hw_attach(dev);
246 	if (error != 0)
247 		return (error);
248 
249 	mtx_init(&sc->lock, "ccp", NULL, MTX_DEF);
250 
251 	ccp_initialize_queues(sc);
252 
253 	if (g_ccp_softc == NULL) {
254 		g_ccp_softc = sc;
255 		if ((sc->hw_features & VERSION_CAP_TRNG) != 0)
256 			random_source_register(&random_ccp);
257 	}
258 
259 	return (0);
260 }
261 
262 static int
263 ccp_detach(device_t dev)
264 {
265 	struct ccp_softc *sc;
266 
267 	sc = device_get_softc(dev);
268 
269 	mtx_lock(&sc->lock);
270 	sc->detaching = true;
271 	mtx_unlock(&sc->lock);
272 
273 	crypto_unregister_all(sc->cid);
274 	if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0)
275 		random_source_deregister(&random_ccp);
276 
277 	ccp_hw_detach(dev);
278 	ccp_free_queues(sc);
279 
280 	if (g_ccp_softc == sc)
281 		g_ccp_softc = NULL;
282 
283 	mtx_destroy(&sc->lock);
284 	return (0);
285 }
286 
287 static void
288 ccp_init_hmac_digest(struct ccp_session *s, const char *key, int klen)
289 {
290 	union authctx auth_ctx;
291 	struct auth_hash *axf;
292 	u_int i;
293 
294 	/*
295 	 * If the key is larger than the block size, use the digest of
296 	 * the key as the key instead.
297 	 */
298 	axf = s->hmac.auth_hash;
299 	if (klen > axf->blocksize) {
300 		axf->Init(&auth_ctx);
301 		axf->Update(&auth_ctx, key, klen);
302 		axf->Final(s->hmac.ipad, &auth_ctx);
303 		explicit_bzero(&auth_ctx, sizeof(auth_ctx));
304 		klen = axf->hashsize;
305 	} else
306 		memcpy(s->hmac.ipad, key, klen);
307 
308 	memset(s->hmac.ipad + klen, 0, axf->blocksize - klen);
309 	memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize);
310 
311 	for (i = 0; i < axf->blocksize; i++) {
312 		s->hmac.ipad[i] ^= HMAC_IPAD_VAL;
313 		s->hmac.opad[i] ^= HMAC_OPAD_VAL;
314 	}
315 }
316 
317 static bool
318 ccp_aes_check_keylen(int alg, int klen)
319 {
320 
321 	switch (klen * 8) {
322 	case 128:
323 	case 192:
324 		if (alg == CRYPTO_AES_XTS)
325 			return (false);
326 		break;
327 	case 256:
328 		break;
329 	case 512:
330 		if (alg != CRYPTO_AES_XTS)
331 			return (false);
332 		break;
333 	default:
334 		return (false);
335 	}
336 	return (true);
337 }
338 
339 static void
340 ccp_aes_setkey(struct ccp_session *s, int alg, const void *key, int klen)
341 {
342 	unsigned kbits;
343 
344 	if (alg == CRYPTO_AES_XTS)
345 		kbits = (klen / 2) * 8;
346 	else
347 		kbits = klen * 8;
348 
349 	switch (kbits) {
350 	case 128:
351 		s->blkcipher.cipher_type = CCP_AES_TYPE_128;
352 		break;
353 	case 192:
354 		s->blkcipher.cipher_type = CCP_AES_TYPE_192;
355 		break;
356 	case 256:
357 		s->blkcipher.cipher_type = CCP_AES_TYPE_256;
358 		break;
359 	default:
360 		panic("should not get here");
361 	}
362 
363 	s->blkcipher.key_len = klen;
364 	memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
365 }
366 
367 static bool
368 ccp_auth_supported(struct ccp_softc *sc,
369     const struct crypto_session_params *csp)
370 {
371 
372 	if ((sc->hw_features & VERSION_CAP_SHA) == 0)
373 		return (false);
374 	switch (csp->csp_auth_alg) {
375 	case CRYPTO_SHA1_HMAC:
376 	case CRYPTO_SHA2_256_HMAC:
377 	case CRYPTO_SHA2_384_HMAC:
378 	case CRYPTO_SHA2_512_HMAC:
379 		if (csp->csp_auth_key == NULL)
380 			return (false);
381 		break;
382 	default:
383 		return (false);
384 	}
385 	return (true);
386 }
387 
388 static bool
389 ccp_cipher_supported(struct ccp_softc *sc,
390     const struct crypto_session_params *csp)
391 {
392 
393 	if ((sc->hw_features & VERSION_CAP_AES) == 0)
394 		return (false);
395 	switch (csp->csp_cipher_alg) {
396 	case CRYPTO_AES_CBC:
397 		if (csp->csp_ivlen != AES_BLOCK_LEN)
398 			return (false);
399 		break;
400 	case CRYPTO_AES_ICM:
401 		if (csp->csp_ivlen != AES_BLOCK_LEN)
402 			return (false);
403 		break;
404 	case CRYPTO_AES_XTS:
405 		if (csp->csp_ivlen != AES_XTS_IV_LEN)
406 			return (false);
407 		break;
408 	default:
409 		return (false);
410 	}
411 	return (ccp_aes_check_keylen(csp->csp_cipher_alg,
412 	    csp->csp_cipher_klen));
413 }
414 
415 static int
416 ccp_probesession(device_t dev, const struct crypto_session_params *csp)
417 {
418 	struct ccp_softc *sc;
419 
420 	if (csp->csp_flags != 0)
421 		return (EINVAL);
422 	sc = device_get_softc(dev);
423 	switch (csp->csp_mode) {
424 	case CSP_MODE_DIGEST:
425 		if (!ccp_auth_supported(sc, csp))
426 			return (EINVAL);
427 		break;
428 	case CSP_MODE_CIPHER:
429 		if (!ccp_cipher_supported(sc, csp))
430 			return (EINVAL);
431 		break;
432 	case CSP_MODE_AEAD:
433 		switch (csp->csp_cipher_alg) {
434 		case CRYPTO_AES_NIST_GCM_16:
435 			if (csp->csp_ivlen != AES_GCM_IV_LEN)
436 				return (EINVAL);
437 			if (csp->csp_auth_mlen < 0 ||
438 			    csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
439 				return (EINVAL);
440 			if ((sc->hw_features & VERSION_CAP_AES) == 0)
441 				return (EINVAL);
442 			break;
443 		default:
444 			return (EINVAL);
445 		}
446 		break;
447 	case CSP_MODE_ETA:
448 		if (!ccp_auth_supported(sc, csp) ||
449 		    !ccp_cipher_supported(sc, csp))
450 			return (EINVAL);
451 		break;
452 	default:
453 		return (EINVAL);
454 	}
455 
456 	return (CRYPTODEV_PROBE_HARDWARE);
457 }
458 
459 static int
460 ccp_newsession(device_t dev, crypto_session_t cses,
461     const struct crypto_session_params *csp)
462 {
463 	struct ccp_softc *sc;
464 	struct ccp_session *s;
465 	struct auth_hash *auth_hash;
466 	enum ccp_aes_mode cipher_mode;
467 	unsigned auth_mode;
468 	unsigned q;
469 
470 	/* XXX reconcile auth_mode with use by ccp_sha */
471 	switch (csp->csp_auth_alg) {
472 	case CRYPTO_SHA1_HMAC:
473 		auth_hash = &auth_hash_hmac_sha1;
474 		auth_mode = SHA1;
475 		break;
476 	case CRYPTO_SHA2_256_HMAC:
477 		auth_hash = &auth_hash_hmac_sha2_256;
478 		auth_mode = SHA2_256;
479 		break;
480 	case CRYPTO_SHA2_384_HMAC:
481 		auth_hash = &auth_hash_hmac_sha2_384;
482 		auth_mode = SHA2_384;
483 		break;
484 	case CRYPTO_SHA2_512_HMAC:
485 		auth_hash = &auth_hash_hmac_sha2_512;
486 		auth_mode = SHA2_512;
487 		break;
488 	default:
489 		auth_hash = NULL;
490 		auth_mode = 0;
491 		break;
492 	}
493 
494 	switch (csp->csp_cipher_alg) {
495 	case CRYPTO_AES_CBC:
496 		cipher_mode = CCP_AES_MODE_CBC;
497 		break;
498 	case CRYPTO_AES_ICM:
499 		cipher_mode = CCP_AES_MODE_CTR;
500 		break;
501 	case CRYPTO_AES_NIST_GCM_16:
502 		cipher_mode = CCP_AES_MODE_GCTR;
503 		break;
504 	case CRYPTO_AES_XTS:
505 		cipher_mode = CCP_AES_MODE_XTS;
506 		break;
507 	default:
508 		cipher_mode = CCP_AES_MODE_ECB;
509 		break;
510 	}
511 
512 	sc = device_get_softc(dev);
513 	mtx_lock(&sc->lock);
514 	if (sc->detaching) {
515 		mtx_unlock(&sc->lock);
516 		return (ENXIO);
517 	}
518 
519 	s = crypto_get_driver_session(cses);
520 
521 	/* Just grab the first usable queue for now. */
522 	for (q = 0; q < nitems(sc->queues); q++)
523 		if ((sc->valid_queues & (1 << q)) != 0)
524 			break;
525 	if (q == nitems(sc->queues)) {
526 		mtx_unlock(&sc->lock);
527 		return (ENXIO);
528 	}
529 	s->queue = q;
530 
531 	switch (csp->csp_mode) {
532 	case CSP_MODE_AEAD:
533 		s->mode = GCM;
534 		break;
535 	case CSP_MODE_ETA:
536 		s->mode = AUTHENC;
537 		break;
538 	case CSP_MODE_DIGEST:
539 		s->mode = HMAC;
540 		break;
541 	case CSP_MODE_CIPHER:
542 		s->mode = BLKCIPHER;
543 		break;
544 	}
545 
546 	if (s->mode == GCM) {
547 		if (csp->csp_auth_mlen == 0)
548 			s->gmac.hash_len = AES_GMAC_HASH_LEN;
549 		else
550 			s->gmac.hash_len = csp->csp_auth_mlen;
551 	} else if (auth_hash != NULL) {
552 		s->hmac.auth_hash = auth_hash;
553 		s->hmac.auth_mode = auth_mode;
554 		if (csp->csp_auth_mlen == 0)
555 			s->hmac.hash_len = auth_hash->hashsize;
556 		else
557 			s->hmac.hash_len = csp->csp_auth_mlen;
558 		ccp_init_hmac_digest(s, csp->csp_auth_key, csp->csp_auth_klen);
559 	}
560 	if (cipher_mode != CCP_AES_MODE_ECB) {
561 		s->blkcipher.cipher_mode = cipher_mode;
562 		if (csp->csp_cipher_key != NULL)
563 			ccp_aes_setkey(s, csp->csp_cipher_alg,
564 			    csp->csp_cipher_key, csp->csp_cipher_klen);
565 	}
566 
567 	s->active = true;
568 	mtx_unlock(&sc->lock);
569 
570 	return (0);
571 }
572 
573 static void
574 ccp_freesession(device_t dev, crypto_session_t cses)
575 {
576 	struct ccp_session *s;
577 
578 	s = crypto_get_driver_session(cses);
579 
580 	if (s->pending != 0)
581 		device_printf(dev,
582 		    "session %p freed with %d pending requests\n", s,
583 		    s->pending);
584 	s->active = false;
585 }
586 
587 static int
588 ccp_process(device_t dev, struct cryptop *crp, int hint)
589 {
590 	const struct crypto_session_params *csp;
591 	struct ccp_softc *sc;
592 	struct ccp_queue *qp;
593 	struct ccp_session *s;
594 	int error;
595 	bool qpheld;
596 
597 	qpheld = false;
598 	qp = NULL;
599 
600 	csp = crypto_get_params(crp->crp_session);
601 	s = crypto_get_driver_session(crp->crp_session);
602 	sc = device_get_softc(dev);
603 	mtx_lock(&sc->lock);
604 	qp = &sc->queues[s->queue];
605 	mtx_unlock(&sc->lock);
606 	error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT);
607 	if (error != 0)
608 		goto out;
609 	qpheld = true;
610 
611 	error = ccp_populate_sglist(qp->cq_sg_crp, crp);
612 	if (error != 0)
613 		goto out;
614 
615 	if (crp->crp_auth_key != NULL) {
616 		KASSERT(s->hmac.auth_hash != NULL, ("auth key without HMAC"));
617 		ccp_init_hmac_digest(s, crp->crp_auth_key, csp->csp_auth_klen);
618 	}
619 	if (crp->crp_cipher_key != NULL)
620 		ccp_aes_setkey(s, csp->csp_cipher_alg, crp->crp_cipher_key,
621 		    csp->csp_cipher_klen);
622 
623 	switch (s->mode) {
624 	case HMAC:
625 		if (s->pending != 0) {
626 			error = EAGAIN;
627 			break;
628 		}
629 		error = ccp_hmac(qp, s, crp);
630 		break;
631 	case BLKCIPHER:
632 		if (s->pending != 0) {
633 			error = EAGAIN;
634 			break;
635 		}
636 		error = ccp_blkcipher(qp, s, crp);
637 		break;
638 	case AUTHENC:
639 		if (s->pending != 0) {
640 			error = EAGAIN;
641 			break;
642 		}
643 		error = ccp_authenc(qp, s, crp);
644 		break;
645 	case GCM:
646 		if (crp->crp_payload_length == 0) {
647 			mtx_unlock(&qp->cq_lock);
648 			ccp_gcm_soft(s, crp);
649 			return (0);
650 		}
651 		if (s->pending != 0) {
652 			error = EAGAIN;
653 			break;
654 		}
655 		error = ccp_gcm(qp, s, crp);
656 		break;
657 	}
658 
659 	if (error == 0)
660 		s->pending++;
661 
662 out:
663 	if (qpheld) {
664 		if (error != 0) {
665 			/*
666 			 * Squash EAGAIN so callers don't uselessly and
667 			 * expensively retry if the ring was full.
668 			 */
669 			if (error == EAGAIN)
670 				error = ENOMEM;
671 			ccp_queue_abort(qp);
672 		} else
673 			ccp_queue_release(qp);
674 	}
675 
676 	if (error != 0) {
677 		DPRINTF(dev, "%s: early error:%d\n", __func__, error);
678 		crp->crp_etype = error;
679 		crypto_done(crp);
680 	}
681 	return (0);
682 }
683 
684 static device_method_t ccp_methods[] = {
685 	DEVMETHOD(device_probe,		ccp_probe),
686 	DEVMETHOD(device_attach,	ccp_attach),
687 	DEVMETHOD(device_detach,	ccp_detach),
688 
689 	DEVMETHOD(cryptodev_probesession, ccp_probesession),
690 	DEVMETHOD(cryptodev_newsession,	ccp_newsession),
691 	DEVMETHOD(cryptodev_freesession, ccp_freesession),
692 	DEVMETHOD(cryptodev_process,	ccp_process),
693 
694 	DEVMETHOD_END
695 };
696 
697 static driver_t ccp_driver = {
698 	"ccp",
699 	ccp_methods,
700 	sizeof(struct ccp_softc)
701 };
702 
703 static devclass_t ccp_devclass;
704 DRIVER_MODULE(ccp, pci, ccp_driver, ccp_devclass, NULL, NULL);
705 MODULE_VERSION(ccp, 1);
706 MODULE_DEPEND(ccp, crypto, 1, 1, 1);
707 MODULE_DEPEND(ccp, random_device, 1, 1, 1);
708 #if 0	/* There are enough known issues that we shouldn't load automatically */
709 MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids,
710     nitems(ccp_ids));
711 #endif
712 
713 static int
714 ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags)
715 {
716 	struct ccp_softc *sc;
717 
718 	mtx_assert(&qp->cq_lock, MA_OWNED);
719 	sc = qp->cq_softc;
720 
721 	if (n < 1 || n >= (1 << sc->ring_size_order))
722 		return (EINVAL);
723 
724 	while (true) {
725 		if (ccp_queue_get_ring_space(qp) >= n)
726 			return (0);
727 		if ((mflags & M_WAITOK) == 0)
728 			return (EAGAIN);
729 		qp->cq_waiting = true;
730 		msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0);
731 	}
732 }
733 
734 int
735 ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags)
736 {
737 	int error;
738 
739 	mtx_lock(&qp->cq_lock);
740 	qp->cq_acq_tail = qp->cq_tail;
741 	error = ccp_queue_reserve_space(qp, n, mflags);
742 	if (error != 0)
743 		mtx_unlock(&qp->cq_lock);
744 	return (error);
745 }
746 
747 void
748 ccp_queue_release(struct ccp_queue *qp)
749 {
750 
751 	mtx_assert(&qp->cq_lock, MA_OWNED);
752 	if (qp->cq_tail != qp->cq_acq_tail) {
753 		wmb();
754 		ccp_queue_write_tail(qp);
755 	}
756 	mtx_unlock(&qp->cq_lock);
757 }
758 
759 void
760 ccp_queue_abort(struct ccp_queue *qp)
761 {
762 	unsigned i;
763 
764 	mtx_assert(&qp->cq_lock, MA_OWNED);
765 
766 	/* Wipe out any descriptors associated with this aborted txn. */
767 	for (i = qp->cq_acq_tail; i != qp->cq_tail;
768 	    i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) {
769 		memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i]));
770 	}
771 	qp->cq_tail = qp->cq_acq_tail;
772 
773 	mtx_unlock(&qp->cq_lock);
774 }
775 
776 #ifdef DDB
777 #define	_db_show_lock(lo)	LOCK_CLASS(lo)->lc_ddb_show(lo)
778 #define	db_show_lock(lk)	_db_show_lock(&(lk)->lock_object)
779 static void
780 db_show_ccp_sc(struct ccp_softc *sc)
781 {
782 
783 	db_printf("ccp softc at %p\n", sc);
784 	db_printf(" cid: %d\n", (int)sc->cid);
785 
786 	db_printf(" lock: ");
787 	db_show_lock(&sc->lock);
788 
789 	db_printf(" detaching: %d\n", (int)sc->detaching);
790 	db_printf(" ring_size_order: %u\n", sc->ring_size_order);
791 
792 	db_printf(" hw_version: %d\n", (int)sc->hw_version);
793 	db_printf(" hw_features: %b\n", (int)sc->hw_features,
794 	    "\20\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA"
795 	    "\11SHA\0103DES\07AES");
796 
797 	db_printf(" hw status:\n");
798 	db_ccp_show_hw(sc);
799 }
800 
801 static void
802 db_show_ccp_qp(struct ccp_queue *qp)
803 {
804 
805 	db_printf(" lock: ");
806 	db_show_lock(&qp->cq_lock);
807 
808 	db_printf(" cq_qindex: %u\n", qp->cq_qindex);
809 	db_printf(" cq_softc: %p\n", qp->cq_softc);
810 
811 	db_printf(" head: %u\n", qp->cq_head);
812 	db_printf(" tail: %u\n", qp->cq_tail);
813 	db_printf(" acq_tail: %u\n", qp->cq_acq_tail);
814 	db_printf(" desc_ring: %p\n", qp->desc_ring);
815 	db_printf(" completions_ring: %p\n", qp->completions_ring);
816 	db_printf(" descriptors (phys): 0x%jx\n",
817 	    (uintmax_t)qp->desc_ring_bus_addr);
818 
819 	db_printf(" hw status:\n");
820 	db_ccp_show_queue_hw(qp);
821 }
822 
823 DB_SHOW_COMMAND(ccp, db_show_ccp)
824 {
825 	struct ccp_softc *sc;
826 	unsigned unit, qindex;
827 
828 	if (!have_addr)
829 		goto usage;
830 
831 	unit = (unsigned)addr;
832 
833 	sc = devclass_get_softc(ccp_devclass, unit);
834 	if (sc == NULL) {
835 		db_printf("No such device ccp%u\n", unit);
836 		goto usage;
837 	}
838 
839 	if (count == -1) {
840 		db_show_ccp_sc(sc);
841 		return;
842 	}
843 
844 	qindex = (unsigned)count;
845 	if (qindex >= nitems(sc->queues)) {
846 		db_printf("No such queue %u\n", qindex);
847 		goto usage;
848 	}
849 	db_show_ccp_qp(&sc->queues[qindex]);
850 	return;
851 
852 usage:
853 	db_printf("usage: show ccp <unit>[,<qindex>]\n");
854 	return;
855 }
856 #endif /* DDB */
857