xref: /freebsd/sys/crypto/ccp/ccp.c (revision e8d8bef961a50d4dc22501cde4fb9fb0be1b2532) 
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2017 Chelsio Communications, Inc.
5  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
6  * All rights reserved.
7  * Largely borrowed from ccr(4), Written by: John Baldwin <jhb@FreeBSD.org>
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include "opt_ddb.h"
35 
36 #include <sys/param.h>
37 #include <sys/bus.h>
38 #include <sys/lock.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/mutex.h>
42 #include <sys/module.h>
43 #include <sys/random.h>
44 #include <sys/sglist.h>
45 #include <sys/sysctl.h>
46 
47 #ifdef DDB
48 #include <ddb/ddb.h>
49 #endif
50 
51 #include <dev/pci/pcivar.h>
52 
53 #include <dev/random/randomdev.h>
54 
55 #include <opencrypto/cryptodev.h>
56 #include <opencrypto/xform.h>
57 
58 #include "cryptodev_if.h"
59 
60 #include "ccp.h"
61 #include "ccp_hardware.h"
62 
63 MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto");
64 
65 /*
66  * Need a global softc available for garbage random_source API, which lacks any
67  * context pointer.  It's also handy for debugging.
68  */
69 struct ccp_softc *g_ccp_softc;
70 
71 bool g_debug_print = false;
72 SYSCTL_BOOL(_hw_ccp, OID_AUTO, debug, CTLFLAG_RWTUN, &g_debug_print, 0,
73     "Set to enable debugging log messages");
74 
75 static struct pciid {
76 	uint32_t devid;
77 	const char *desc;
78 } ccp_ids[] = {
79 	{ 0x14561022, "AMD CCP-5a" },
80 	{ 0x14681022, "AMD CCP-5b" },
81 	{ 0x15df1022, "AMD CCP-5a" },
82 };
83 
84 static struct random_source random_ccp = {
85 	.rs_ident = "AMD CCP TRNG",
86 	.rs_source = RANDOM_PURE_CCP,
87 	.rs_read = random_ccp_read,
88 };
89 
90 /*
91  * ccp_populate_sglist() generates a scatter/gather list that covers the entire
92  * crypto operation buffer.
93  */
94 static int
95 ccp_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
96 {
97 	int error;
98 
99 	sglist_reset(sg);
100 	switch (cb->cb_type) {
101 	case CRYPTO_BUF_MBUF:
102 		error = sglist_append_mbuf(sg, cb->cb_mbuf);
103 		break;
104 	case CRYPTO_BUF_SINGLE_MBUF:
105 		error = sglist_append_single_mbuf(sg, cb->cb_mbuf);
106 		break;
107 	case CRYPTO_BUF_UIO:
108 		error = sglist_append_uio(sg, cb->cb_uio);
109 		break;
110 	case CRYPTO_BUF_CONTIG:
111 		error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len);
112 		break;
113 	case CRYPTO_BUF_VMPAGE:
114 		error = sglist_append_vmpages(sg, cb->cb_vm_page,
115 		    cb->cb_vm_page_len, cb->cb_vm_page_offset);
116 		break;
117 	default:
118 		error = EINVAL;
119 	}
120 	return (error);
121 }
122 
123 static int
124 ccp_probe(device_t dev)
125 {
126 	struct pciid *ip;
127 	uint32_t id;
128 
129 	id = pci_get_devid(dev);
130 	for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) {
131 		if (id == ip->devid) {
132 			device_set_desc(dev, ip->desc);
133 			return (0);
134 		}
135 	}
136 	return (ENXIO);
137 }
138 
139 static void
140 ccp_initialize_queues(struct ccp_softc *sc)
141 {
142 	struct ccp_queue *qp;
143 	size_t i;
144 
145 	for (i = 0; i < nitems(sc->queues); i++) {
146 		qp = &sc->queues[i];
147 
148 		qp->cq_softc = sc;
149 		qp->cq_qindex = i;
150 		mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF);
151 		/* XXX - arbitrarily chosen sizes */
152 		qp->cq_sg_crp = sglist_alloc(32, M_WAITOK);
153 		/* Two more SGEs than sg_crp to accommodate ipad. */
154 		qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK);
155 		qp->cq_sg_dst = sglist_alloc(2, M_WAITOK);
156 	}
157 }
158 
159 static void
160 ccp_free_queues(struct ccp_softc *sc)
161 {
162 	struct ccp_queue *qp;
163 	size_t i;
164 
165 	for (i = 0; i < nitems(sc->queues); i++) {
166 		qp = &sc->queues[i];
167 
168 		mtx_destroy(&qp->cq_lock);
169 		sglist_free(qp->cq_sg_crp);
170 		sglist_free(qp->cq_sg_ulptx);
171 		sglist_free(qp->cq_sg_dst);
172 	}
173 }
174 
175 static int
176 ccp_attach(device_t dev)
177 {
178 	struct ccp_softc *sc;
179 	int error;
180 
181 	sc = device_get_softc(dev);
182 	sc->dev = dev;
183 
184 	sc->cid = crypto_get_driverid(dev, sizeof(struct ccp_session),
185 	    CRYPTOCAP_F_HARDWARE);
186 	if (sc->cid < 0) {
187 		device_printf(dev, "could not get crypto driver id\n");
188 		return (ENXIO);
189 	}
190 
191 	error = ccp_hw_attach(dev);
192 	if (error != 0)
193 		return (error);
194 
195 	mtx_init(&sc->lock, "ccp", NULL, MTX_DEF);
196 
197 	ccp_initialize_queues(sc);
198 
199 	if (g_ccp_softc == NULL) {
200 		g_ccp_softc = sc;
201 		if ((sc->hw_features & VERSION_CAP_TRNG) != 0)
202 			random_source_register(&random_ccp);
203 	}
204 
205 	return (0);
206 }
207 
208 static int
209 ccp_detach(device_t dev)
210 {
211 	struct ccp_softc *sc;
212 
213 	sc = device_get_softc(dev);
214 
215 	mtx_lock(&sc->lock);
216 	sc->detaching = true;
217 	mtx_unlock(&sc->lock);
218 
219 	crypto_unregister_all(sc->cid);
220 	if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0)
221 		random_source_deregister(&random_ccp);
222 
223 	ccp_hw_detach(dev);
224 	ccp_free_queues(sc);
225 
226 	if (g_ccp_softc == sc)
227 		g_ccp_softc = NULL;
228 
229 	mtx_destroy(&sc->lock);
230 	return (0);
231 }
232 
233 static void
234 ccp_init_hmac_digest(struct ccp_session *s, const char *key, int klen)
235 {
236 	union authctx auth_ctx;
237 	struct auth_hash *axf;
238 	u_int i;
239 
240 	/*
241 	 * If the key is larger than the block size, use the digest of
242 	 * the key as the key instead.
243 	 */
244 	axf = s->hmac.auth_hash;
245 	if (klen > axf->blocksize) {
246 		axf->Init(&auth_ctx);
247 		axf->Update(&auth_ctx, key, klen);
248 		axf->Final(s->hmac.ipad, &auth_ctx);
249 		explicit_bzero(&auth_ctx, sizeof(auth_ctx));
250 		klen = axf->hashsize;
251 	} else
252 		memcpy(s->hmac.ipad, key, klen);
253 
254 	memset(s->hmac.ipad + klen, 0, axf->blocksize - klen);
255 	memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize);
256 
257 	for (i = 0; i < axf->blocksize; i++) {
258 		s->hmac.ipad[i] ^= HMAC_IPAD_VAL;
259 		s->hmac.opad[i] ^= HMAC_OPAD_VAL;
260 	}
261 }
262 
263 static bool
264 ccp_aes_check_keylen(int alg, int klen)
265 {
266 
267 	switch (klen * 8) {
268 	case 128:
269 	case 192:
270 		if (alg == CRYPTO_AES_XTS)
271 			return (false);
272 		break;
273 	case 256:
274 		break;
275 	case 512:
276 		if (alg != CRYPTO_AES_XTS)
277 			return (false);
278 		break;
279 	default:
280 		return (false);
281 	}
282 	return (true);
283 }
284 
285 static void
286 ccp_aes_setkey(struct ccp_session *s, int alg, const void *key, int klen)
287 {
288 	unsigned kbits;
289 
290 	if (alg == CRYPTO_AES_XTS)
291 		kbits = (klen / 2) * 8;
292 	else
293 		kbits = klen * 8;
294 
295 	switch (kbits) {
296 	case 128:
297 		s->blkcipher.cipher_type = CCP_AES_TYPE_128;
298 		break;
299 	case 192:
300 		s->blkcipher.cipher_type = CCP_AES_TYPE_192;
301 		break;
302 	case 256:
303 		s->blkcipher.cipher_type = CCP_AES_TYPE_256;
304 		break;
305 	default:
306 		panic("should not get here");
307 	}
308 
309 	s->blkcipher.key_len = klen;
310 	memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
311 }
312 
313 static bool
314 ccp_auth_supported(struct ccp_softc *sc,
315     const struct crypto_session_params *csp)
316 {
317 
318 	if ((sc->hw_features & VERSION_CAP_SHA) == 0)
319 		return (false);
320 	switch (csp->csp_auth_alg) {
321 	case CRYPTO_SHA1_HMAC:
322 	case CRYPTO_SHA2_256_HMAC:
323 	case CRYPTO_SHA2_384_HMAC:
324 	case CRYPTO_SHA2_512_HMAC:
325 		if (csp->csp_auth_key == NULL)
326 			return (false);
327 		break;
328 	default:
329 		return (false);
330 	}
331 	return (true);
332 }
333 
334 static bool
335 ccp_cipher_supported(struct ccp_softc *sc,
336     const struct crypto_session_params *csp)
337 {
338 
339 	if ((sc->hw_features & VERSION_CAP_AES) == 0)
340 		return (false);
341 	switch (csp->csp_cipher_alg) {
342 	case CRYPTO_AES_CBC:
343 		if (csp->csp_ivlen != AES_BLOCK_LEN)
344 			return (false);
345 		break;
346 	case CRYPTO_AES_ICM:
347 		if (csp->csp_ivlen != AES_BLOCK_LEN)
348 			return (false);
349 		break;
350 	case CRYPTO_AES_XTS:
351 		if (csp->csp_ivlen != AES_XTS_IV_LEN)
352 			return (false);
353 		break;
354 	default:
355 		return (false);
356 	}
357 	return (ccp_aes_check_keylen(csp->csp_cipher_alg,
358 	    csp->csp_cipher_klen));
359 }
360 
361 static int
362 ccp_probesession(device_t dev, const struct crypto_session_params *csp)
363 {
364 	struct ccp_softc *sc;
365 
366 	if (csp->csp_flags != 0)
367 		return (EINVAL);
368 	sc = device_get_softc(dev);
369 	switch (csp->csp_mode) {
370 	case CSP_MODE_DIGEST:
371 		if (!ccp_auth_supported(sc, csp))
372 			return (EINVAL);
373 		break;
374 	case CSP_MODE_CIPHER:
375 		if (!ccp_cipher_supported(sc, csp))
376 			return (EINVAL);
377 		break;
378 	case CSP_MODE_AEAD:
379 		switch (csp->csp_cipher_alg) {
380 		case CRYPTO_AES_NIST_GCM_16:
381 			if (csp->csp_ivlen != AES_GCM_IV_LEN)
382 				return (EINVAL);
383 			if (csp->csp_auth_mlen < 0 ||
384 			    csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
385 				return (EINVAL);
386 			if ((sc->hw_features & VERSION_CAP_AES) == 0)
387 				return (EINVAL);
388 			break;
389 		default:
390 			return (EINVAL);
391 		}
392 		break;
393 	case CSP_MODE_ETA:
394 		if (!ccp_auth_supported(sc, csp) ||
395 		    !ccp_cipher_supported(sc, csp))
396 			return (EINVAL);
397 		break;
398 	default:
399 		return (EINVAL);
400 	}
401 
402 	return (CRYPTODEV_PROBE_HARDWARE);
403 }
404 
405 static int
406 ccp_newsession(device_t dev, crypto_session_t cses,
407     const struct crypto_session_params *csp)
408 {
409 	struct ccp_softc *sc;
410 	struct ccp_session *s;
411 	struct auth_hash *auth_hash;
412 	enum ccp_aes_mode cipher_mode;
413 	unsigned auth_mode;
414 	unsigned q;
415 
416 	/* XXX reconcile auth_mode with use by ccp_sha */
417 	switch (csp->csp_auth_alg) {
418 	case CRYPTO_SHA1_HMAC:
419 		auth_hash = &auth_hash_hmac_sha1;
420 		auth_mode = SHA1;
421 		break;
422 	case CRYPTO_SHA2_256_HMAC:
423 		auth_hash = &auth_hash_hmac_sha2_256;
424 		auth_mode = SHA2_256;
425 		break;
426 	case CRYPTO_SHA2_384_HMAC:
427 		auth_hash = &auth_hash_hmac_sha2_384;
428 		auth_mode = SHA2_384;
429 		break;
430 	case CRYPTO_SHA2_512_HMAC:
431 		auth_hash = &auth_hash_hmac_sha2_512;
432 		auth_mode = SHA2_512;
433 		break;
434 	default:
435 		auth_hash = NULL;
436 		auth_mode = 0;
437 		break;
438 	}
439 
440 	switch (csp->csp_cipher_alg) {
441 	case CRYPTO_AES_CBC:
442 		cipher_mode = CCP_AES_MODE_CBC;
443 		break;
444 	case CRYPTO_AES_ICM:
445 		cipher_mode = CCP_AES_MODE_CTR;
446 		break;
447 	case CRYPTO_AES_NIST_GCM_16:
448 		cipher_mode = CCP_AES_MODE_GCTR;
449 		break;
450 	case CRYPTO_AES_XTS:
451 		cipher_mode = CCP_AES_MODE_XTS;
452 		break;
453 	default:
454 		cipher_mode = CCP_AES_MODE_ECB;
455 		break;
456 	}
457 
458 	sc = device_get_softc(dev);
459 	mtx_lock(&sc->lock);
460 	if (sc->detaching) {
461 		mtx_unlock(&sc->lock);
462 		return (ENXIO);
463 	}
464 
465 	s = crypto_get_driver_session(cses);
466 
467 	/* Just grab the first usable queue for now. */
468 	for (q = 0; q < nitems(sc->queues); q++)
469 		if ((sc->valid_queues & (1 << q)) != 0)
470 			break;
471 	if (q == nitems(sc->queues)) {
472 		mtx_unlock(&sc->lock);
473 		return (ENXIO);
474 	}
475 	s->queue = q;
476 
477 	switch (csp->csp_mode) {
478 	case CSP_MODE_AEAD:
479 		s->mode = GCM;
480 		break;
481 	case CSP_MODE_ETA:
482 		s->mode = AUTHENC;
483 		break;
484 	case CSP_MODE_DIGEST:
485 		s->mode = HMAC;
486 		break;
487 	case CSP_MODE_CIPHER:
488 		s->mode = BLKCIPHER;
489 		break;
490 	}
491 
492 	if (s->mode == GCM) {
493 		if (csp->csp_auth_mlen == 0)
494 			s->gmac.hash_len = AES_GMAC_HASH_LEN;
495 		else
496 			s->gmac.hash_len = csp->csp_auth_mlen;
497 	} else if (auth_hash != NULL) {
498 		s->hmac.auth_hash = auth_hash;
499 		s->hmac.auth_mode = auth_mode;
500 		if (csp->csp_auth_mlen == 0)
501 			s->hmac.hash_len = auth_hash->hashsize;
502 		else
503 			s->hmac.hash_len = csp->csp_auth_mlen;
504 		ccp_init_hmac_digest(s, csp->csp_auth_key, csp->csp_auth_klen);
505 	}
506 	if (cipher_mode != CCP_AES_MODE_ECB) {
507 		s->blkcipher.cipher_mode = cipher_mode;
508 		if (csp->csp_cipher_key != NULL)
509 			ccp_aes_setkey(s, csp->csp_cipher_alg,
510 			    csp->csp_cipher_key, csp->csp_cipher_klen);
511 	}
512 
513 	s->active = true;
514 	mtx_unlock(&sc->lock);
515 
516 	return (0);
517 }
518 
519 static void
520 ccp_freesession(device_t dev, crypto_session_t cses)
521 {
522 	struct ccp_session *s;
523 
524 	s = crypto_get_driver_session(cses);
525 
526 	if (s->pending != 0)
527 		device_printf(dev,
528 		    "session %p freed with %d pending requests\n", s,
529 		    s->pending);
530 	s->active = false;
531 }
532 
533 static int
534 ccp_process(device_t dev, struct cryptop *crp, int hint)
535 {
536 	const struct crypto_session_params *csp;
537 	struct ccp_softc *sc;
538 	struct ccp_queue *qp;
539 	struct ccp_session *s;
540 	int error;
541 	bool qpheld;
542 
543 	qpheld = false;
544 	qp = NULL;
545 
546 	csp = crypto_get_params(crp->crp_session);
547 	s = crypto_get_driver_session(crp->crp_session);
548 	sc = device_get_softc(dev);
549 	mtx_lock(&sc->lock);
550 	qp = &sc->queues[s->queue];
551 	mtx_unlock(&sc->lock);
552 	error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT);
553 	if (error != 0)
554 		goto out;
555 	qpheld = true;
556 
557 	error = ccp_populate_sglist(qp->cq_sg_crp, &crp->crp_buf);
558 	if (error != 0)
559 		goto out;
560 
561 	if (crp->crp_auth_key != NULL) {
562 		KASSERT(s->hmac.auth_hash != NULL, ("auth key without HMAC"));
563 		ccp_init_hmac_digest(s, crp->crp_auth_key, csp->csp_auth_klen);
564 	}
565 	if (crp->crp_cipher_key != NULL)
566 		ccp_aes_setkey(s, csp->csp_cipher_alg, crp->crp_cipher_key,
567 		    csp->csp_cipher_klen);
568 
569 	switch (s->mode) {
570 	case HMAC:
571 		if (s->pending != 0) {
572 			error = EAGAIN;
573 			break;
574 		}
575 		error = ccp_hmac(qp, s, crp);
576 		break;
577 	case BLKCIPHER:
578 		if (s->pending != 0) {
579 			error = EAGAIN;
580 			break;
581 		}
582 		error = ccp_blkcipher(qp, s, crp);
583 		break;
584 	case AUTHENC:
585 		if (s->pending != 0) {
586 			error = EAGAIN;
587 			break;
588 		}
589 		error = ccp_authenc(qp, s, crp);
590 		break;
591 	case GCM:
592 		if (s->pending != 0) {
593 			error = EAGAIN;
594 			break;
595 		}
596 		error = ccp_gcm(qp, s, crp);
597 		break;
598 	}
599 
600 	if (error == 0)
601 		s->pending++;
602 
603 out:
604 	if (qpheld) {
605 		if (error != 0) {
606 			/*
607 			 * Squash EAGAIN so callers don't uselessly and
608 			 * expensively retry if the ring was full.
609 			 */
610 			if (error == EAGAIN)
611 				error = ENOMEM;
612 			ccp_queue_abort(qp);
613 		} else
614 			ccp_queue_release(qp);
615 	}
616 
617 	if (error != 0) {
618 		DPRINTF(dev, "%s: early error:%d\n", __func__, error);
619 		crp->crp_etype = error;
620 		crypto_done(crp);
621 	}
622 	return (0);
623 }
624 
625 static device_method_t ccp_methods[] = {
626 	DEVMETHOD(device_probe,		ccp_probe),
627 	DEVMETHOD(device_attach,	ccp_attach),
628 	DEVMETHOD(device_detach,	ccp_detach),
629 
630 	DEVMETHOD(cryptodev_probesession, ccp_probesession),
631 	DEVMETHOD(cryptodev_newsession,	ccp_newsession),
632 	DEVMETHOD(cryptodev_freesession, ccp_freesession),
633 	DEVMETHOD(cryptodev_process,	ccp_process),
634 
635 	DEVMETHOD_END
636 };
637 
638 static driver_t ccp_driver = {
639 	"ccp",
640 	ccp_methods,
641 	sizeof(struct ccp_softc)
642 };
643 
644 static devclass_t ccp_devclass;
645 DRIVER_MODULE(ccp, pci, ccp_driver, ccp_devclass, NULL, NULL);
646 MODULE_VERSION(ccp, 1);
647 MODULE_DEPEND(ccp, crypto, 1, 1, 1);
648 MODULE_DEPEND(ccp, random_device, 1, 1, 1);
649 #if 0	/* There are enough known issues that we shouldn't load automatically */
650 MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids,
651     nitems(ccp_ids));
652 #endif
653 
654 static int
655 ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags)
656 {
657 	struct ccp_softc *sc;
658 
659 	mtx_assert(&qp->cq_lock, MA_OWNED);
660 	sc = qp->cq_softc;
661 
662 	if (n < 1 || n >= (1 << sc->ring_size_order))
663 		return (EINVAL);
664 
665 	while (true) {
666 		if (ccp_queue_get_ring_space(qp) >= n)
667 			return (0);
668 		if ((mflags & M_WAITOK) == 0)
669 			return (EAGAIN);
670 		qp->cq_waiting = true;
671 		msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0);
672 	}
673 }
674 
675 int
676 ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags)
677 {
678 	int error;
679 
680 	mtx_lock(&qp->cq_lock);
681 	qp->cq_acq_tail = qp->cq_tail;
682 	error = ccp_queue_reserve_space(qp, n, mflags);
683 	if (error != 0)
684 		mtx_unlock(&qp->cq_lock);
685 	return (error);
686 }
687 
688 void
689 ccp_queue_release(struct ccp_queue *qp)
690 {
691 
692 	mtx_assert(&qp->cq_lock, MA_OWNED);
693 	if (qp->cq_tail != qp->cq_acq_tail) {
694 		wmb();
695 		ccp_queue_write_tail(qp);
696 	}
697 	mtx_unlock(&qp->cq_lock);
698 }
699 
700 void
701 ccp_queue_abort(struct ccp_queue *qp)
702 {
703 	unsigned i;
704 
705 	mtx_assert(&qp->cq_lock, MA_OWNED);
706 
707 	/* Wipe out any descriptors associated with this aborted txn. */
708 	for (i = qp->cq_acq_tail; i != qp->cq_tail;
709 	    i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) {
710 		memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i]));
711 	}
712 	qp->cq_tail = qp->cq_acq_tail;
713 
714 	mtx_unlock(&qp->cq_lock);
715 }
716 
717 #ifdef DDB
718 #define	_db_show_lock(lo)	LOCK_CLASS(lo)->lc_ddb_show(lo)
719 #define	db_show_lock(lk)	_db_show_lock(&(lk)->lock_object)
720 static void
721 db_show_ccp_sc(struct ccp_softc *sc)
722 {
723 
724 	db_printf("ccp softc at %p\n", sc);
725 	db_printf(" cid: %d\n", (int)sc->cid);
726 
727 	db_printf(" lock: ");
728 	db_show_lock(&sc->lock);
729 
730 	db_printf(" detaching: %d\n", (int)sc->detaching);
731 	db_printf(" ring_size_order: %u\n", sc->ring_size_order);
732 
733 	db_printf(" hw_version: %d\n", (int)sc->hw_version);
734 	db_printf(" hw_features: %b\n", (int)sc->hw_features,
735 	    "\20\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA"
736 	    "\11SHA\0103DES\07AES");
737 
738 	db_printf(" hw status:\n");
739 	db_ccp_show_hw(sc);
740 }
741 
742 static void
743 db_show_ccp_qp(struct ccp_queue *qp)
744 {
745 
746 	db_printf(" lock: ");
747 	db_show_lock(&qp->cq_lock);
748 
749 	db_printf(" cq_qindex: %u\n", qp->cq_qindex);
750 	db_printf(" cq_softc: %p\n", qp->cq_softc);
751 
752 	db_printf(" head: %u\n", qp->cq_head);
753 	db_printf(" tail: %u\n", qp->cq_tail);
754 	db_printf(" acq_tail: %u\n", qp->cq_acq_tail);
755 	db_printf(" desc_ring: %p\n", qp->desc_ring);
756 	db_printf(" completions_ring: %p\n", qp->completions_ring);
757 	db_printf(" descriptors (phys): 0x%jx\n",
758 	    (uintmax_t)qp->desc_ring_bus_addr);
759 
760 	db_printf(" hw status:\n");
761 	db_ccp_show_queue_hw(qp);
762 }
763 
764 DB_SHOW_COMMAND(ccp, db_show_ccp)
765 {
766 	struct ccp_softc *sc;
767 	unsigned unit, qindex;
768 
769 	if (!have_addr)
770 		goto usage;
771 
772 	unit = (unsigned)addr;
773 
774 	sc = devclass_get_softc(ccp_devclass, unit);
775 	if (sc == NULL) {
776 		db_printf("No such device ccp%u\n", unit);
777 		goto usage;
778 	}
779 
780 	if (count == -1) {
781 		db_show_ccp_sc(sc);
782 		return;
783 	}
784 
785 	qindex = (unsigned)count;
786 	if (qindex >= nitems(sc->queues)) {
787 		db_printf("No such queue %u\n", qindex);
788 		goto usage;
789 	}
790 	db_show_ccp_qp(&sc->queues[qindex]);
791 	return;
792 
793 usage:
794 	db_printf("usage: show ccp <unit>[,<qindex>]\n");
795 	return;
796 }
797 #endif /* DDB */
798