xref: /freebsd/sys/crypto/ccp/ccp.c (revision 89e3d5671ba13dceca272d5b159c9bd805f3f504) 
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2017 Chelsio Communications, Inc.
5  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
6  * All rights reserved.
7  * Largely borrowed from ccr(4), Written by: John Baldwin <jhb@FreeBSD.org>
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include "opt_ddb.h"
35 
36 #include <sys/param.h>
37 #include <sys/bus.h>
38 #include <sys/lock.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/mutex.h>
42 #include <sys/module.h>
43 #include <sys/random.h>
44 #include <sys/sglist.h>
45 #include <sys/sysctl.h>
46 
47 #ifdef DDB
48 #include <ddb/ddb.h>
49 #endif
50 
51 #include <dev/pci/pcivar.h>
52 
53 #include <dev/random/randomdev.h>
54 
55 #include <opencrypto/cryptodev.h>
56 #include <opencrypto/xform.h>
57 
58 #include "cryptodev_if.h"
59 
60 #include "ccp.h"
61 #include "ccp_hardware.h"
62 
63 MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto");
64 
65 /*
66  * Need a global softc available for garbage random_source API, which lacks any
67  * context pointer.  It's also handy for debugging.
68  */
69 struct ccp_softc *g_ccp_softc;
70 
71 bool g_debug_print = false;
72 SYSCTL_BOOL(_hw_ccp, OID_AUTO, debug, CTLFLAG_RWTUN, &g_debug_print, 0,
73     "Set to enable debugging log messages");
74 
75 static struct pciid {
76 	uint32_t devid;
77 	const char *desc;
78 } ccp_ids[] = {
79 	{ 0x14561022, "AMD CCP-5a" },
80 	{ 0x14681022, "AMD CCP-5b" },
81 	{ 0x15df1022, "AMD CCP-5a" },
82 };
83 
84 static struct random_source random_ccp = {
85 	.rs_ident = "AMD CCP TRNG",
86 	.rs_source = RANDOM_PURE_CCP,
87 	.rs_read = random_ccp_read,
88 };
89 
90 /*
91  * ccp_populate_sglist() generates a scatter/gather list that covers the entire
92  * crypto operation buffer.
93  */
94 static int
95 ccp_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
96 {
97 	int error;
98 
99 	sglist_reset(sg);
100 	switch (cb->cb_type) {
101 	case CRYPTO_BUF_MBUF:
102 		error = sglist_append_mbuf(sg, cb->cb_mbuf);
103 		break;
104 	case CRYPTO_BUF_UIO:
105 		error = sglist_append_uio(sg, cb->cb_uio);
106 		break;
107 	case CRYPTO_BUF_CONTIG:
108 		error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len);
109 		break;
110 	case CRYPTO_BUF_VMPAGE:
111 		error = sglist_append_vmpages(sg, cb->cb_vm_page,
112 		    cb->cb_vm_page_len, cb->cb_vm_page_offset);
113 		break;
114 	default:
115 		error = EINVAL;
116 	}
117 	return (error);
118 }
119 
120 static int
121 ccp_probe(device_t dev)
122 {
123 	struct pciid *ip;
124 	uint32_t id;
125 
126 	id = pci_get_devid(dev);
127 	for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) {
128 		if (id == ip->devid) {
129 			device_set_desc(dev, ip->desc);
130 			return (0);
131 		}
132 	}
133 	return (ENXIO);
134 }
135 
136 static void
137 ccp_initialize_queues(struct ccp_softc *sc)
138 {
139 	struct ccp_queue *qp;
140 	size_t i;
141 
142 	for (i = 0; i < nitems(sc->queues); i++) {
143 		qp = &sc->queues[i];
144 
145 		qp->cq_softc = sc;
146 		qp->cq_qindex = i;
147 		mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF);
148 		/* XXX - arbitrarily chosen sizes */
149 		qp->cq_sg_crp = sglist_alloc(32, M_WAITOK);
150 		/* Two more SGEs than sg_crp to accommodate ipad. */
151 		qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK);
152 		qp->cq_sg_dst = sglist_alloc(2, M_WAITOK);
153 	}
154 }
155 
156 static void
157 ccp_free_queues(struct ccp_softc *sc)
158 {
159 	struct ccp_queue *qp;
160 	size_t i;
161 
162 	for (i = 0; i < nitems(sc->queues); i++) {
163 		qp = &sc->queues[i];
164 
165 		mtx_destroy(&qp->cq_lock);
166 		sglist_free(qp->cq_sg_crp);
167 		sglist_free(qp->cq_sg_ulptx);
168 		sglist_free(qp->cq_sg_dst);
169 	}
170 }
171 
172 static int
173 ccp_attach(device_t dev)
174 {
175 	struct ccp_softc *sc;
176 	int error;
177 
178 	sc = device_get_softc(dev);
179 	sc->dev = dev;
180 
181 	sc->cid = crypto_get_driverid(dev, sizeof(struct ccp_session),
182 	    CRYPTOCAP_F_HARDWARE);
183 	if (sc->cid < 0) {
184 		device_printf(dev, "could not get crypto driver id\n");
185 		return (ENXIO);
186 	}
187 
188 	error = ccp_hw_attach(dev);
189 	if (error != 0)
190 		return (error);
191 
192 	mtx_init(&sc->lock, "ccp", NULL, MTX_DEF);
193 
194 	ccp_initialize_queues(sc);
195 
196 	if (g_ccp_softc == NULL) {
197 		g_ccp_softc = sc;
198 		if ((sc->hw_features & VERSION_CAP_TRNG) != 0)
199 			random_source_register(&random_ccp);
200 	}
201 
202 	return (0);
203 }
204 
205 static int
206 ccp_detach(device_t dev)
207 {
208 	struct ccp_softc *sc;
209 
210 	sc = device_get_softc(dev);
211 
212 	mtx_lock(&sc->lock);
213 	sc->detaching = true;
214 	mtx_unlock(&sc->lock);
215 
216 	crypto_unregister_all(sc->cid);
217 	if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0)
218 		random_source_deregister(&random_ccp);
219 
220 	ccp_hw_detach(dev);
221 	ccp_free_queues(sc);
222 
223 	if (g_ccp_softc == sc)
224 		g_ccp_softc = NULL;
225 
226 	mtx_destroy(&sc->lock);
227 	return (0);
228 }
229 
230 static void
231 ccp_init_hmac_digest(struct ccp_session *s, const char *key, int klen)
232 {
233 	union authctx auth_ctx;
234 	struct auth_hash *axf;
235 	u_int i;
236 
237 	/*
238 	 * If the key is larger than the block size, use the digest of
239 	 * the key as the key instead.
240 	 */
241 	axf = s->hmac.auth_hash;
242 	if (klen > axf->blocksize) {
243 		axf->Init(&auth_ctx);
244 		axf->Update(&auth_ctx, key, klen);
245 		axf->Final(s->hmac.ipad, &auth_ctx);
246 		explicit_bzero(&auth_ctx, sizeof(auth_ctx));
247 		klen = axf->hashsize;
248 	} else
249 		memcpy(s->hmac.ipad, key, klen);
250 
251 	memset(s->hmac.ipad + klen, 0, axf->blocksize - klen);
252 	memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize);
253 
254 	for (i = 0; i < axf->blocksize; i++) {
255 		s->hmac.ipad[i] ^= HMAC_IPAD_VAL;
256 		s->hmac.opad[i] ^= HMAC_OPAD_VAL;
257 	}
258 }
259 
260 static bool
261 ccp_aes_check_keylen(int alg, int klen)
262 {
263 
264 	switch (klen * 8) {
265 	case 128:
266 	case 192:
267 		if (alg == CRYPTO_AES_XTS)
268 			return (false);
269 		break;
270 	case 256:
271 		break;
272 	case 512:
273 		if (alg != CRYPTO_AES_XTS)
274 			return (false);
275 		break;
276 	default:
277 		return (false);
278 	}
279 	return (true);
280 }
281 
282 static void
283 ccp_aes_setkey(struct ccp_session *s, int alg, const void *key, int klen)
284 {
285 	unsigned kbits;
286 
287 	if (alg == CRYPTO_AES_XTS)
288 		kbits = (klen / 2) * 8;
289 	else
290 		kbits = klen * 8;
291 
292 	switch (kbits) {
293 	case 128:
294 		s->blkcipher.cipher_type = CCP_AES_TYPE_128;
295 		break;
296 	case 192:
297 		s->blkcipher.cipher_type = CCP_AES_TYPE_192;
298 		break;
299 	case 256:
300 		s->blkcipher.cipher_type = CCP_AES_TYPE_256;
301 		break;
302 	default:
303 		panic("should not get here");
304 	}
305 
306 	s->blkcipher.key_len = klen;
307 	memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
308 }
309 
310 static bool
311 ccp_auth_supported(struct ccp_softc *sc,
312     const struct crypto_session_params *csp)
313 {
314 
315 	if ((sc->hw_features & VERSION_CAP_SHA) == 0)
316 		return (false);
317 	switch (csp->csp_auth_alg) {
318 	case CRYPTO_SHA1_HMAC:
319 	case CRYPTO_SHA2_256_HMAC:
320 	case CRYPTO_SHA2_384_HMAC:
321 	case CRYPTO_SHA2_512_HMAC:
322 		if (csp->csp_auth_key == NULL)
323 			return (false);
324 		break;
325 	default:
326 		return (false);
327 	}
328 	return (true);
329 }
330 
331 static bool
332 ccp_cipher_supported(struct ccp_softc *sc,
333     const struct crypto_session_params *csp)
334 {
335 
336 	if ((sc->hw_features & VERSION_CAP_AES) == 0)
337 		return (false);
338 	switch (csp->csp_cipher_alg) {
339 	case CRYPTO_AES_CBC:
340 		if (csp->csp_ivlen != AES_BLOCK_LEN)
341 			return (false);
342 		break;
343 	case CRYPTO_AES_ICM:
344 		if (csp->csp_ivlen != AES_BLOCK_LEN)
345 			return (false);
346 		break;
347 	case CRYPTO_AES_XTS:
348 		if (csp->csp_ivlen != AES_XTS_IV_LEN)
349 			return (false);
350 		break;
351 	default:
352 		return (false);
353 	}
354 	return (ccp_aes_check_keylen(csp->csp_cipher_alg,
355 	    csp->csp_cipher_klen));
356 }
357 
358 static int
359 ccp_probesession(device_t dev, const struct crypto_session_params *csp)
360 {
361 	struct ccp_softc *sc;
362 
363 	if (csp->csp_flags != 0)
364 		return (EINVAL);
365 	sc = device_get_softc(dev);
366 	switch (csp->csp_mode) {
367 	case CSP_MODE_DIGEST:
368 		if (!ccp_auth_supported(sc, csp))
369 			return (EINVAL);
370 		break;
371 	case CSP_MODE_CIPHER:
372 		if (!ccp_cipher_supported(sc, csp))
373 			return (EINVAL);
374 		break;
375 	case CSP_MODE_AEAD:
376 		switch (csp->csp_cipher_alg) {
377 		case CRYPTO_AES_NIST_GCM_16:
378 			if (csp->csp_ivlen != AES_GCM_IV_LEN)
379 				return (EINVAL);
380 			if (csp->csp_auth_mlen < 0 ||
381 			    csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
382 				return (EINVAL);
383 			if ((sc->hw_features & VERSION_CAP_AES) == 0)
384 				return (EINVAL);
385 			break;
386 		default:
387 			return (EINVAL);
388 		}
389 		break;
390 	case CSP_MODE_ETA:
391 		if (!ccp_auth_supported(sc, csp) ||
392 		    !ccp_cipher_supported(sc, csp))
393 			return (EINVAL);
394 		break;
395 	default:
396 		return (EINVAL);
397 	}
398 
399 	return (CRYPTODEV_PROBE_HARDWARE);
400 }
401 
402 static int
403 ccp_newsession(device_t dev, crypto_session_t cses,
404     const struct crypto_session_params *csp)
405 {
406 	struct ccp_softc *sc;
407 	struct ccp_session *s;
408 	struct auth_hash *auth_hash;
409 	enum ccp_aes_mode cipher_mode;
410 	unsigned auth_mode;
411 	unsigned q;
412 
413 	/* XXX reconcile auth_mode with use by ccp_sha */
414 	switch (csp->csp_auth_alg) {
415 	case CRYPTO_SHA1_HMAC:
416 		auth_hash = &auth_hash_hmac_sha1;
417 		auth_mode = SHA1;
418 		break;
419 	case CRYPTO_SHA2_256_HMAC:
420 		auth_hash = &auth_hash_hmac_sha2_256;
421 		auth_mode = SHA2_256;
422 		break;
423 	case CRYPTO_SHA2_384_HMAC:
424 		auth_hash = &auth_hash_hmac_sha2_384;
425 		auth_mode = SHA2_384;
426 		break;
427 	case CRYPTO_SHA2_512_HMAC:
428 		auth_hash = &auth_hash_hmac_sha2_512;
429 		auth_mode = SHA2_512;
430 		break;
431 	default:
432 		auth_hash = NULL;
433 		auth_mode = 0;
434 		break;
435 	}
436 
437 	switch (csp->csp_cipher_alg) {
438 	case CRYPTO_AES_CBC:
439 		cipher_mode = CCP_AES_MODE_CBC;
440 		break;
441 	case CRYPTO_AES_ICM:
442 		cipher_mode = CCP_AES_MODE_CTR;
443 		break;
444 	case CRYPTO_AES_NIST_GCM_16:
445 		cipher_mode = CCP_AES_MODE_GCTR;
446 		break;
447 	case CRYPTO_AES_XTS:
448 		cipher_mode = CCP_AES_MODE_XTS;
449 		break;
450 	default:
451 		cipher_mode = CCP_AES_MODE_ECB;
452 		break;
453 	}
454 
455 	sc = device_get_softc(dev);
456 	mtx_lock(&sc->lock);
457 	if (sc->detaching) {
458 		mtx_unlock(&sc->lock);
459 		return (ENXIO);
460 	}
461 
462 	s = crypto_get_driver_session(cses);
463 
464 	/* Just grab the first usable queue for now. */
465 	for (q = 0; q < nitems(sc->queues); q++)
466 		if ((sc->valid_queues & (1 << q)) != 0)
467 			break;
468 	if (q == nitems(sc->queues)) {
469 		mtx_unlock(&sc->lock);
470 		return (ENXIO);
471 	}
472 	s->queue = q;
473 
474 	switch (csp->csp_mode) {
475 	case CSP_MODE_AEAD:
476 		s->mode = GCM;
477 		break;
478 	case CSP_MODE_ETA:
479 		s->mode = AUTHENC;
480 		break;
481 	case CSP_MODE_DIGEST:
482 		s->mode = HMAC;
483 		break;
484 	case CSP_MODE_CIPHER:
485 		s->mode = BLKCIPHER;
486 		break;
487 	}
488 
489 	if (s->mode == GCM) {
490 		if (csp->csp_auth_mlen == 0)
491 			s->gmac.hash_len = AES_GMAC_HASH_LEN;
492 		else
493 			s->gmac.hash_len = csp->csp_auth_mlen;
494 	} else if (auth_hash != NULL) {
495 		s->hmac.auth_hash = auth_hash;
496 		s->hmac.auth_mode = auth_mode;
497 		if (csp->csp_auth_mlen == 0)
498 			s->hmac.hash_len = auth_hash->hashsize;
499 		else
500 			s->hmac.hash_len = csp->csp_auth_mlen;
501 		ccp_init_hmac_digest(s, csp->csp_auth_key, csp->csp_auth_klen);
502 	}
503 	if (cipher_mode != CCP_AES_MODE_ECB) {
504 		s->blkcipher.cipher_mode = cipher_mode;
505 		if (csp->csp_cipher_key != NULL)
506 			ccp_aes_setkey(s, csp->csp_cipher_alg,
507 			    csp->csp_cipher_key, csp->csp_cipher_klen);
508 	}
509 
510 	s->active = true;
511 	mtx_unlock(&sc->lock);
512 
513 	return (0);
514 }
515 
516 static void
517 ccp_freesession(device_t dev, crypto_session_t cses)
518 {
519 	struct ccp_session *s;
520 
521 	s = crypto_get_driver_session(cses);
522 
523 	if (s->pending != 0)
524 		device_printf(dev,
525 		    "session %p freed with %d pending requests\n", s,
526 		    s->pending);
527 	s->active = false;
528 }
529 
530 static int
531 ccp_process(device_t dev, struct cryptop *crp, int hint)
532 {
533 	const struct crypto_session_params *csp;
534 	struct ccp_softc *sc;
535 	struct ccp_queue *qp;
536 	struct ccp_session *s;
537 	int error;
538 	bool qpheld;
539 
540 	qpheld = false;
541 	qp = NULL;
542 
543 	csp = crypto_get_params(crp->crp_session);
544 	s = crypto_get_driver_session(crp->crp_session);
545 	sc = device_get_softc(dev);
546 	mtx_lock(&sc->lock);
547 	qp = &sc->queues[s->queue];
548 	mtx_unlock(&sc->lock);
549 	error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT);
550 	if (error != 0)
551 		goto out;
552 	qpheld = true;
553 
554 	error = ccp_populate_sglist(qp->cq_sg_crp, &crp->crp_buf);
555 	if (error != 0)
556 		goto out;
557 
558 	if (crp->crp_auth_key != NULL) {
559 		KASSERT(s->hmac.auth_hash != NULL, ("auth key without HMAC"));
560 		ccp_init_hmac_digest(s, crp->crp_auth_key, csp->csp_auth_klen);
561 	}
562 	if (crp->crp_cipher_key != NULL)
563 		ccp_aes_setkey(s, csp->csp_cipher_alg, crp->crp_cipher_key,
564 		    csp->csp_cipher_klen);
565 
566 	switch (s->mode) {
567 	case HMAC:
568 		if (s->pending != 0) {
569 			error = EAGAIN;
570 			break;
571 		}
572 		error = ccp_hmac(qp, s, crp);
573 		break;
574 	case BLKCIPHER:
575 		if (s->pending != 0) {
576 			error = EAGAIN;
577 			break;
578 		}
579 		error = ccp_blkcipher(qp, s, crp);
580 		break;
581 	case AUTHENC:
582 		if (s->pending != 0) {
583 			error = EAGAIN;
584 			break;
585 		}
586 		error = ccp_authenc(qp, s, crp);
587 		break;
588 	case GCM:
589 		if (s->pending != 0) {
590 			error = EAGAIN;
591 			break;
592 		}
593 		error = ccp_gcm(qp, s, crp);
594 		break;
595 	}
596 
597 	if (error == 0)
598 		s->pending++;
599 
600 out:
601 	if (qpheld) {
602 		if (error != 0) {
603 			/*
604 			 * Squash EAGAIN so callers don't uselessly and
605 			 * expensively retry if the ring was full.
606 			 */
607 			if (error == EAGAIN)
608 				error = ENOMEM;
609 			ccp_queue_abort(qp);
610 		} else
611 			ccp_queue_release(qp);
612 	}
613 
614 	if (error != 0) {
615 		DPRINTF(dev, "%s: early error:%d\n", __func__, error);
616 		crp->crp_etype = error;
617 		crypto_done(crp);
618 	}
619 	return (0);
620 }
621 
622 static device_method_t ccp_methods[] = {
623 	DEVMETHOD(device_probe,		ccp_probe),
624 	DEVMETHOD(device_attach,	ccp_attach),
625 	DEVMETHOD(device_detach,	ccp_detach),
626 
627 	DEVMETHOD(cryptodev_probesession, ccp_probesession),
628 	DEVMETHOD(cryptodev_newsession,	ccp_newsession),
629 	DEVMETHOD(cryptodev_freesession, ccp_freesession),
630 	DEVMETHOD(cryptodev_process,	ccp_process),
631 
632 	DEVMETHOD_END
633 };
634 
635 static driver_t ccp_driver = {
636 	"ccp",
637 	ccp_methods,
638 	sizeof(struct ccp_softc)
639 };
640 
641 static devclass_t ccp_devclass;
642 DRIVER_MODULE(ccp, pci, ccp_driver, ccp_devclass, NULL, NULL);
643 MODULE_VERSION(ccp, 1);
644 MODULE_DEPEND(ccp, crypto, 1, 1, 1);
645 MODULE_DEPEND(ccp, random_device, 1, 1, 1);
646 #if 0	/* There are enough known issues that we shouldn't load automatically */
647 MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids,
648     nitems(ccp_ids));
649 #endif
650 
651 static int
652 ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags)
653 {
654 	struct ccp_softc *sc;
655 
656 	mtx_assert(&qp->cq_lock, MA_OWNED);
657 	sc = qp->cq_softc;
658 
659 	if (n < 1 || n >= (1 << sc->ring_size_order))
660 		return (EINVAL);
661 
662 	while (true) {
663 		if (ccp_queue_get_ring_space(qp) >= n)
664 			return (0);
665 		if ((mflags & M_WAITOK) == 0)
666 			return (EAGAIN);
667 		qp->cq_waiting = true;
668 		msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0);
669 	}
670 }
671 
672 int
673 ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags)
674 {
675 	int error;
676 
677 	mtx_lock(&qp->cq_lock);
678 	qp->cq_acq_tail = qp->cq_tail;
679 	error = ccp_queue_reserve_space(qp, n, mflags);
680 	if (error != 0)
681 		mtx_unlock(&qp->cq_lock);
682 	return (error);
683 }
684 
685 void
686 ccp_queue_release(struct ccp_queue *qp)
687 {
688 
689 	mtx_assert(&qp->cq_lock, MA_OWNED);
690 	if (qp->cq_tail != qp->cq_acq_tail) {
691 		wmb();
692 		ccp_queue_write_tail(qp);
693 	}
694 	mtx_unlock(&qp->cq_lock);
695 }
696 
697 void
698 ccp_queue_abort(struct ccp_queue *qp)
699 {
700 	unsigned i;
701 
702 	mtx_assert(&qp->cq_lock, MA_OWNED);
703 
704 	/* Wipe out any descriptors associated with this aborted txn. */
705 	for (i = qp->cq_acq_tail; i != qp->cq_tail;
706 	    i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) {
707 		memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i]));
708 	}
709 	qp->cq_tail = qp->cq_acq_tail;
710 
711 	mtx_unlock(&qp->cq_lock);
712 }
713 
714 #ifdef DDB
715 #define	_db_show_lock(lo)	LOCK_CLASS(lo)->lc_ddb_show(lo)
716 #define	db_show_lock(lk)	_db_show_lock(&(lk)->lock_object)
717 static void
718 db_show_ccp_sc(struct ccp_softc *sc)
719 {
720 
721 	db_printf("ccp softc at %p\n", sc);
722 	db_printf(" cid: %d\n", (int)sc->cid);
723 
724 	db_printf(" lock: ");
725 	db_show_lock(&sc->lock);
726 
727 	db_printf(" detaching: %d\n", (int)sc->detaching);
728 	db_printf(" ring_size_order: %u\n", sc->ring_size_order);
729 
730 	db_printf(" hw_version: %d\n", (int)sc->hw_version);
731 	db_printf(" hw_features: %b\n", (int)sc->hw_features,
732 	    "\20\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA"
733 	    "\11SHA\0103DES\07AES");
734 
735 	db_printf(" hw status:\n");
736 	db_ccp_show_hw(sc);
737 }
738 
739 static void
740 db_show_ccp_qp(struct ccp_queue *qp)
741 {
742 
743 	db_printf(" lock: ");
744 	db_show_lock(&qp->cq_lock);
745 
746 	db_printf(" cq_qindex: %u\n", qp->cq_qindex);
747 	db_printf(" cq_softc: %p\n", qp->cq_softc);
748 
749 	db_printf(" head: %u\n", qp->cq_head);
750 	db_printf(" tail: %u\n", qp->cq_tail);
751 	db_printf(" acq_tail: %u\n", qp->cq_acq_tail);
752 	db_printf(" desc_ring: %p\n", qp->desc_ring);
753 	db_printf(" completions_ring: %p\n", qp->completions_ring);
754 	db_printf(" descriptors (phys): 0x%jx\n",
755 	    (uintmax_t)qp->desc_ring_bus_addr);
756 
757 	db_printf(" hw status:\n");
758 	db_ccp_show_queue_hw(qp);
759 }
760 
761 DB_SHOW_COMMAND(ccp, db_show_ccp)
762 {
763 	struct ccp_softc *sc;
764 	unsigned unit, qindex;
765 
766 	if (!have_addr)
767 		goto usage;
768 
769 	unit = (unsigned)addr;
770 
771 	sc = devclass_get_softc(ccp_devclass, unit);
772 	if (sc == NULL) {
773 		db_printf("No such device ccp%u\n", unit);
774 		goto usage;
775 	}
776 
777 	if (count == -1) {
778 		db_show_ccp_sc(sc);
779 		return;
780 	}
781 
782 	qindex = (unsigned)count;
783 	if (qindex >= nitems(sc->queues)) {
784 		db_printf("No such queue %u\n", qindex);
785 		goto usage;
786 	}
787 	db_show_ccp_qp(&sc->queues[qindex]);
788 	return;
789 
790 usage:
791 	db_printf("usage: show ccp <unit>[,<qindex>]\n");
792 	return;
793 }
794 #endif /* DDB */
795