1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2017 Chelsio Communications, Inc. 5 * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org> 6 * All rights reserved. 7 * Largely borrowed from ccr(4), Written by: John Baldwin <jhb@FreeBSD.org> 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_ddb.h" 35 36 #include <sys/param.h> 37 #include <sys/bus.h> 38 #include <sys/lock.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/mutex.h> 42 #include <sys/module.h> 43 #include <sys/random.h> 44 #include <sys/sglist.h> 45 #include <sys/sysctl.h> 46 47 #ifdef DDB 48 #include <ddb/ddb.h> 49 #endif 50 51 #include <dev/pci/pcivar.h> 52 53 #include <dev/random/randomdev.h> 54 55 #include <opencrypto/cryptodev.h> 56 #include <opencrypto/xform.h> 57 58 #include "cryptodev_if.h" 59 60 #include "ccp.h" 61 #include "ccp_hardware.h" 62 63 MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto"); 64 65 /* 66 * Need a global softc available for garbage random_source API, which lacks any 67 * context pointer. It's also handy for debugging. 68 */ 69 struct ccp_softc *g_ccp_softc; 70 71 bool g_debug_print = false; 72 SYSCTL_BOOL(_hw_ccp, OID_AUTO, debug, CTLFLAG_RWTUN, &g_debug_print, 0, 73 "Set to enable debugging log messages"); 74 75 static struct pciid { 76 uint32_t devid; 77 const char *desc; 78 } ccp_ids[] = { 79 { 0x14561022, "AMD CCP-5a" }, 80 { 0x14681022, "AMD CCP-5b" }, 81 { 0x15df1022, "AMD CCP-5a" }, 82 }; 83 84 static struct random_source random_ccp = { 85 .rs_ident = "AMD CCP TRNG", 86 .rs_source = RANDOM_PURE_CCP, 87 .rs_read = random_ccp_read, 88 }; 89 90 /* 91 * ccp_populate_sglist() generates a scatter/gather list that covers the entire 92 * crypto operation buffer. 93 */ 94 static int 95 ccp_populate_sglist(struct sglist *sg, struct cryptop *crp) 96 { 97 int error; 98 99 sglist_reset(sg); 100 switch (crp->crp_buf_type) { 101 case CRYPTO_BUF_MBUF: 102 error = sglist_append_mbuf(sg, crp->crp_mbuf); 103 break; 104 case CRYPTO_BUF_UIO: 105 error = sglist_append_uio(sg, crp->crp_uio); 106 break; 107 case CRYPTO_BUF_CONTIG: 108 error = sglist_append(sg, crp->crp_buf, crp->crp_ilen); 109 break; 110 default: 111 error = EINVAL; 112 } 113 return (error); 114 } 115 116 /* 117 * Handle a GCM request with an empty payload by performing the 118 * operation in software. 119 */ 120 static void 121 ccp_gcm_soft(struct ccp_session *s, struct cryptop *crp) 122 { 123 struct aes_gmac_ctx gmac_ctx; 124 char block[GMAC_BLOCK_LEN]; 125 char digest[GMAC_DIGEST_LEN]; 126 char iv[AES_BLOCK_LEN]; 127 int i, len; 128 129 /* 130 * This assumes a 12-byte IV from the crp. See longer comment 131 * above in ccp_gcm() for more details. 132 */ 133 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) { 134 crp->crp_etype = EINVAL; 135 goto out; 136 } 137 memcpy(iv, crp->crp_iv, 12); 138 *(uint32_t *)&iv[12] = htobe32(1); 139 140 /* Initialize the MAC. */ 141 AES_GMAC_Init(&gmac_ctx); 142 AES_GMAC_Setkey(&gmac_ctx, s->blkcipher.enckey, s->blkcipher.key_len); 143 AES_GMAC_Reinit(&gmac_ctx, iv, sizeof(iv)); 144 145 /* MAC the AAD. */ 146 for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) { 147 len = imin(crp->crp_aad_length - i, sizeof(block)); 148 crypto_copydata(crp, crp->crp_aad_start + i, len, block); 149 bzero(block + len, sizeof(block) - len); 150 AES_GMAC_Update(&gmac_ctx, block, sizeof(block)); 151 } 152 153 /* Length block. */ 154 bzero(block, sizeof(block)); 155 ((uint32_t *)block)[1] = htobe32(crp->crp_aad_length * 8); 156 AES_GMAC_Update(&gmac_ctx, block, sizeof(block)); 157 AES_GMAC_Final(digest, &gmac_ctx); 158 159 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 160 crypto_copyback(crp, crp->crp_digest_start, sizeof(digest), 161 digest); 162 crp->crp_etype = 0; 163 } else { 164 char digest2[GMAC_DIGEST_LEN]; 165 166 crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2), 167 digest2); 168 if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) 169 crp->crp_etype = 0; 170 else 171 crp->crp_etype = EBADMSG; 172 } 173 out: 174 crypto_done(crp); 175 } 176 177 static int 178 ccp_probe(device_t dev) 179 { 180 struct pciid *ip; 181 uint32_t id; 182 183 id = pci_get_devid(dev); 184 for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) { 185 if (id == ip->devid) { 186 device_set_desc(dev, ip->desc); 187 return (0); 188 } 189 } 190 return (ENXIO); 191 } 192 193 static void 194 ccp_initialize_queues(struct ccp_softc *sc) 195 { 196 struct ccp_queue *qp; 197 size_t i; 198 199 for (i = 0; i < nitems(sc->queues); i++) { 200 qp = &sc->queues[i]; 201 202 qp->cq_softc = sc; 203 qp->cq_qindex = i; 204 mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF); 205 /* XXX - arbitrarily chosen sizes */ 206 qp->cq_sg_crp = sglist_alloc(32, M_WAITOK); 207 /* Two more SGEs than sg_crp to accommodate ipad. */ 208 qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK); 209 qp->cq_sg_dst = sglist_alloc(2, M_WAITOK); 210 } 211 } 212 213 static void 214 ccp_free_queues(struct ccp_softc *sc) 215 { 216 struct ccp_queue *qp; 217 size_t i; 218 219 for (i = 0; i < nitems(sc->queues); i++) { 220 qp = &sc->queues[i]; 221 222 mtx_destroy(&qp->cq_lock); 223 sglist_free(qp->cq_sg_crp); 224 sglist_free(qp->cq_sg_ulptx); 225 sglist_free(qp->cq_sg_dst); 226 } 227 } 228 229 static int 230 ccp_attach(device_t dev) 231 { 232 struct ccp_softc *sc; 233 int error; 234 235 sc = device_get_softc(dev); 236 sc->dev = dev; 237 238 sc->cid = crypto_get_driverid(dev, sizeof(struct ccp_session), 239 CRYPTOCAP_F_HARDWARE); 240 if (sc->cid < 0) { 241 device_printf(dev, "could not get crypto driver id\n"); 242 return (ENXIO); 243 } 244 245 error = ccp_hw_attach(dev); 246 if (error != 0) 247 return (error); 248 249 mtx_init(&sc->lock, "ccp", NULL, MTX_DEF); 250 251 ccp_initialize_queues(sc); 252 253 if (g_ccp_softc == NULL) { 254 g_ccp_softc = sc; 255 if ((sc->hw_features & VERSION_CAP_TRNG) != 0) 256 random_source_register(&random_ccp); 257 } 258 259 return (0); 260 } 261 262 static int 263 ccp_detach(device_t dev) 264 { 265 struct ccp_softc *sc; 266 267 sc = device_get_softc(dev); 268 269 mtx_lock(&sc->lock); 270 sc->detaching = true; 271 mtx_unlock(&sc->lock); 272 273 crypto_unregister_all(sc->cid); 274 if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0) 275 random_source_deregister(&random_ccp); 276 277 ccp_hw_detach(dev); 278 ccp_free_queues(sc); 279 280 if (g_ccp_softc == sc) 281 g_ccp_softc = NULL; 282 283 mtx_destroy(&sc->lock); 284 return (0); 285 } 286 287 static void 288 ccp_init_hmac_digest(struct ccp_session *s, const char *key, int klen) 289 { 290 union authctx auth_ctx; 291 struct auth_hash *axf; 292 u_int i; 293 294 /* 295 * If the key is larger than the block size, use the digest of 296 * the key as the key instead. 297 */ 298 axf = s->hmac.auth_hash; 299 if (klen > axf->blocksize) { 300 axf->Init(&auth_ctx); 301 axf->Update(&auth_ctx, key, klen); 302 axf->Final(s->hmac.ipad, &auth_ctx); 303 explicit_bzero(&auth_ctx, sizeof(auth_ctx)); 304 klen = axf->hashsize; 305 } else 306 memcpy(s->hmac.ipad, key, klen); 307 308 memset(s->hmac.ipad + klen, 0, axf->blocksize - klen); 309 memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize); 310 311 for (i = 0; i < axf->blocksize; i++) { 312 s->hmac.ipad[i] ^= HMAC_IPAD_VAL; 313 s->hmac.opad[i] ^= HMAC_OPAD_VAL; 314 } 315 } 316 317 static bool 318 ccp_aes_check_keylen(int alg, int klen) 319 { 320 321 switch (klen * 8) { 322 case 128: 323 case 192: 324 if (alg == CRYPTO_AES_XTS) 325 return (false); 326 break; 327 case 256: 328 break; 329 case 512: 330 if (alg != CRYPTO_AES_XTS) 331 return (false); 332 break; 333 default: 334 return (false); 335 } 336 return (true); 337 } 338 339 static void 340 ccp_aes_setkey(struct ccp_session *s, int alg, const void *key, int klen) 341 { 342 unsigned kbits; 343 344 if (alg == CRYPTO_AES_XTS) 345 kbits = (klen / 2) * 8; 346 else 347 kbits = klen * 8; 348 349 switch (kbits) { 350 case 128: 351 s->blkcipher.cipher_type = CCP_AES_TYPE_128; 352 break; 353 case 192: 354 s->blkcipher.cipher_type = CCP_AES_TYPE_192; 355 break; 356 case 256: 357 s->blkcipher.cipher_type = CCP_AES_TYPE_256; 358 break; 359 default: 360 panic("should not get here"); 361 } 362 363 s->blkcipher.key_len = klen; 364 memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len); 365 } 366 367 static bool 368 ccp_auth_supported(struct ccp_softc *sc, 369 const struct crypto_session_params *csp) 370 { 371 372 if ((sc->hw_features & VERSION_CAP_SHA) == 0) 373 return (false); 374 switch (csp->csp_auth_alg) { 375 case CRYPTO_SHA1_HMAC: 376 case CRYPTO_SHA2_256_HMAC: 377 case CRYPTO_SHA2_384_HMAC: 378 case CRYPTO_SHA2_512_HMAC: 379 if (csp->csp_auth_key == NULL) 380 return (false); 381 break; 382 default: 383 return (false); 384 } 385 return (true); 386 } 387 388 static bool 389 ccp_cipher_supported(struct ccp_softc *sc, 390 const struct crypto_session_params *csp) 391 { 392 393 if ((sc->hw_features & VERSION_CAP_AES) == 0) 394 return (false); 395 switch (csp->csp_cipher_alg) { 396 case CRYPTO_AES_CBC: 397 if (csp->csp_ivlen != AES_BLOCK_LEN) 398 return (false); 399 break; 400 case CRYPTO_AES_ICM: 401 if (csp->csp_ivlen != AES_BLOCK_LEN) 402 return (false); 403 break; 404 case CRYPTO_AES_XTS: 405 if (csp->csp_ivlen != AES_XTS_IV_LEN) 406 return (false); 407 break; 408 default: 409 return (false); 410 } 411 return (ccp_aes_check_keylen(csp->csp_cipher_alg, 412 csp->csp_cipher_klen)); 413 } 414 415 static int 416 ccp_probesession(device_t dev, const struct crypto_session_params *csp) 417 { 418 struct ccp_softc *sc; 419 420 if (csp->csp_flags != 0) 421 return (EINVAL); 422 sc = device_get_softc(dev); 423 switch (csp->csp_mode) { 424 case CSP_MODE_DIGEST: 425 if (!ccp_auth_supported(sc, csp)) 426 return (EINVAL); 427 break; 428 case CSP_MODE_CIPHER: 429 if (!ccp_cipher_supported(sc, csp)) 430 return (EINVAL); 431 break; 432 case CSP_MODE_AEAD: 433 switch (csp->csp_cipher_alg) { 434 case CRYPTO_AES_NIST_GCM_16: 435 if (csp->csp_ivlen != AES_GCM_IV_LEN) 436 return (EINVAL); 437 if (csp->csp_auth_mlen < 0 || 438 csp->csp_auth_mlen > AES_GMAC_HASH_LEN) 439 return (EINVAL); 440 if ((sc->hw_features & VERSION_CAP_AES) == 0) 441 return (EINVAL); 442 break; 443 default: 444 return (EINVAL); 445 } 446 break; 447 case CSP_MODE_ETA: 448 if (!ccp_auth_supported(sc, csp) || 449 !ccp_cipher_supported(sc, csp)) 450 return (EINVAL); 451 break; 452 default: 453 return (EINVAL); 454 } 455 456 return (CRYPTODEV_PROBE_HARDWARE); 457 } 458 459 static int 460 ccp_newsession(device_t dev, crypto_session_t cses, 461 const struct crypto_session_params *csp) 462 { 463 struct ccp_softc *sc; 464 struct ccp_session *s; 465 struct auth_hash *auth_hash; 466 enum ccp_aes_mode cipher_mode; 467 unsigned auth_mode; 468 unsigned q; 469 470 /* XXX reconcile auth_mode with use by ccp_sha */ 471 switch (csp->csp_auth_alg) { 472 case CRYPTO_SHA1_HMAC: 473 auth_hash = &auth_hash_hmac_sha1; 474 auth_mode = SHA1; 475 break; 476 case CRYPTO_SHA2_256_HMAC: 477 auth_hash = &auth_hash_hmac_sha2_256; 478 auth_mode = SHA2_256; 479 break; 480 case CRYPTO_SHA2_384_HMAC: 481 auth_hash = &auth_hash_hmac_sha2_384; 482 auth_mode = SHA2_384; 483 break; 484 case CRYPTO_SHA2_512_HMAC: 485 auth_hash = &auth_hash_hmac_sha2_512; 486 auth_mode = SHA2_512; 487 break; 488 default: 489 auth_hash = NULL; 490 auth_mode = 0; 491 break; 492 } 493 494 switch (csp->csp_cipher_alg) { 495 case CRYPTO_AES_CBC: 496 cipher_mode = CCP_AES_MODE_CBC; 497 break; 498 case CRYPTO_AES_ICM: 499 cipher_mode = CCP_AES_MODE_CTR; 500 break; 501 case CRYPTO_AES_NIST_GCM_16: 502 cipher_mode = CCP_AES_MODE_GCTR; 503 break; 504 case CRYPTO_AES_XTS: 505 cipher_mode = CCP_AES_MODE_XTS; 506 break; 507 default: 508 cipher_mode = CCP_AES_MODE_ECB; 509 break; 510 } 511 512 sc = device_get_softc(dev); 513 mtx_lock(&sc->lock); 514 if (sc->detaching) { 515 mtx_unlock(&sc->lock); 516 return (ENXIO); 517 } 518 519 s = crypto_get_driver_session(cses); 520 521 /* Just grab the first usable queue for now. */ 522 for (q = 0; q < nitems(sc->queues); q++) 523 if ((sc->valid_queues & (1 << q)) != 0) 524 break; 525 if (q == nitems(sc->queues)) { 526 mtx_unlock(&sc->lock); 527 return (ENXIO); 528 } 529 s->queue = q; 530 531 switch (csp->csp_mode) { 532 case CSP_MODE_AEAD: 533 s->mode = GCM; 534 break; 535 case CSP_MODE_ETA: 536 s->mode = AUTHENC; 537 break; 538 case CSP_MODE_DIGEST: 539 s->mode = HMAC; 540 break; 541 case CSP_MODE_CIPHER: 542 s->mode = BLKCIPHER; 543 break; 544 } 545 546 if (s->mode == GCM) { 547 if (csp->csp_auth_mlen == 0) 548 s->gmac.hash_len = AES_GMAC_HASH_LEN; 549 else 550 s->gmac.hash_len = csp->csp_auth_mlen; 551 } else if (auth_hash != NULL) { 552 s->hmac.auth_hash = auth_hash; 553 s->hmac.auth_mode = auth_mode; 554 if (csp->csp_auth_mlen == 0) 555 s->hmac.hash_len = auth_hash->hashsize; 556 else 557 s->hmac.hash_len = csp->csp_auth_mlen; 558 ccp_init_hmac_digest(s, csp->csp_auth_key, csp->csp_auth_klen); 559 } 560 if (cipher_mode != CCP_AES_MODE_ECB) { 561 s->blkcipher.cipher_mode = cipher_mode; 562 if (csp->csp_cipher_key != NULL) 563 ccp_aes_setkey(s, csp->csp_cipher_alg, 564 csp->csp_cipher_key, csp->csp_cipher_klen); 565 } 566 567 s->active = true; 568 mtx_unlock(&sc->lock); 569 570 return (0); 571 } 572 573 static void 574 ccp_freesession(device_t dev, crypto_session_t cses) 575 { 576 struct ccp_session *s; 577 578 s = crypto_get_driver_session(cses); 579 580 if (s->pending != 0) 581 device_printf(dev, 582 "session %p freed with %d pending requests\n", s, 583 s->pending); 584 s->active = false; 585 } 586 587 static int 588 ccp_process(device_t dev, struct cryptop *crp, int hint) 589 { 590 const struct crypto_session_params *csp; 591 struct ccp_softc *sc; 592 struct ccp_queue *qp; 593 struct ccp_session *s; 594 int error; 595 bool qpheld; 596 597 qpheld = false; 598 qp = NULL; 599 600 csp = crypto_get_params(crp->crp_session); 601 s = crypto_get_driver_session(crp->crp_session); 602 sc = device_get_softc(dev); 603 mtx_lock(&sc->lock); 604 qp = &sc->queues[s->queue]; 605 mtx_unlock(&sc->lock); 606 error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT); 607 if (error != 0) 608 goto out; 609 qpheld = true; 610 611 error = ccp_populate_sglist(qp->cq_sg_crp, crp); 612 if (error != 0) 613 goto out; 614 615 if (crp->crp_auth_key != NULL) { 616 KASSERT(s->hmac.auth_hash != NULL, ("auth key without HMAC")); 617 ccp_init_hmac_digest(s, crp->crp_auth_key, csp->csp_auth_klen); 618 } 619 if (crp->crp_cipher_key != NULL) 620 ccp_aes_setkey(s, csp->csp_cipher_alg, crp->crp_cipher_key, 621 csp->csp_cipher_klen); 622 623 switch (s->mode) { 624 case HMAC: 625 if (s->pending != 0) { 626 error = EAGAIN; 627 break; 628 } 629 error = ccp_hmac(qp, s, crp); 630 break; 631 case BLKCIPHER: 632 if (s->pending != 0) { 633 error = EAGAIN; 634 break; 635 } 636 error = ccp_blkcipher(qp, s, crp); 637 break; 638 case AUTHENC: 639 if (s->pending != 0) { 640 error = EAGAIN; 641 break; 642 } 643 error = ccp_authenc(qp, s, crp); 644 break; 645 case GCM: 646 if (crp->crp_payload_length == 0) { 647 mtx_unlock(&qp->cq_lock); 648 ccp_gcm_soft(s, crp); 649 return (0); 650 } 651 if (s->pending != 0) { 652 error = EAGAIN; 653 break; 654 } 655 error = ccp_gcm(qp, s, crp); 656 break; 657 } 658 659 if (error == 0) 660 s->pending++; 661 662 out: 663 if (qpheld) { 664 if (error != 0) { 665 /* 666 * Squash EAGAIN so callers don't uselessly and 667 * expensively retry if the ring was full. 668 */ 669 if (error == EAGAIN) 670 error = ENOMEM; 671 ccp_queue_abort(qp); 672 } else 673 ccp_queue_release(qp); 674 } 675 676 if (error != 0) { 677 DPRINTF(dev, "%s: early error:%d\n", __func__, error); 678 crp->crp_etype = error; 679 crypto_done(crp); 680 } 681 return (0); 682 } 683 684 static device_method_t ccp_methods[] = { 685 DEVMETHOD(device_probe, ccp_probe), 686 DEVMETHOD(device_attach, ccp_attach), 687 DEVMETHOD(device_detach, ccp_detach), 688 689 DEVMETHOD(cryptodev_probesession, ccp_probesession), 690 DEVMETHOD(cryptodev_newsession, ccp_newsession), 691 DEVMETHOD(cryptodev_freesession, ccp_freesession), 692 DEVMETHOD(cryptodev_process, ccp_process), 693 694 DEVMETHOD_END 695 }; 696 697 static driver_t ccp_driver = { 698 "ccp", 699 ccp_methods, 700 sizeof(struct ccp_softc) 701 }; 702 703 static devclass_t ccp_devclass; 704 DRIVER_MODULE(ccp, pci, ccp_driver, ccp_devclass, NULL, NULL); 705 MODULE_VERSION(ccp, 1); 706 MODULE_DEPEND(ccp, crypto, 1, 1, 1); 707 MODULE_DEPEND(ccp, random_device, 1, 1, 1); 708 #if 0 /* There are enough known issues that we shouldn't load automatically */ 709 MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids, 710 nitems(ccp_ids)); 711 #endif 712 713 static int 714 ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags) 715 { 716 struct ccp_softc *sc; 717 718 mtx_assert(&qp->cq_lock, MA_OWNED); 719 sc = qp->cq_softc; 720 721 if (n < 1 || n >= (1 << sc->ring_size_order)) 722 return (EINVAL); 723 724 while (true) { 725 if (ccp_queue_get_ring_space(qp) >= n) 726 return (0); 727 if ((mflags & M_WAITOK) == 0) 728 return (EAGAIN); 729 qp->cq_waiting = true; 730 msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0); 731 } 732 } 733 734 int 735 ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags) 736 { 737 int error; 738 739 mtx_lock(&qp->cq_lock); 740 qp->cq_acq_tail = qp->cq_tail; 741 error = ccp_queue_reserve_space(qp, n, mflags); 742 if (error != 0) 743 mtx_unlock(&qp->cq_lock); 744 return (error); 745 } 746 747 void 748 ccp_queue_release(struct ccp_queue *qp) 749 { 750 751 mtx_assert(&qp->cq_lock, MA_OWNED); 752 if (qp->cq_tail != qp->cq_acq_tail) { 753 wmb(); 754 ccp_queue_write_tail(qp); 755 } 756 mtx_unlock(&qp->cq_lock); 757 } 758 759 void 760 ccp_queue_abort(struct ccp_queue *qp) 761 { 762 unsigned i; 763 764 mtx_assert(&qp->cq_lock, MA_OWNED); 765 766 /* Wipe out any descriptors associated with this aborted txn. */ 767 for (i = qp->cq_acq_tail; i != qp->cq_tail; 768 i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) { 769 memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i])); 770 } 771 qp->cq_tail = qp->cq_acq_tail; 772 773 mtx_unlock(&qp->cq_lock); 774 } 775 776 #ifdef DDB 777 #define _db_show_lock(lo) LOCK_CLASS(lo)->lc_ddb_show(lo) 778 #define db_show_lock(lk) _db_show_lock(&(lk)->lock_object) 779 static void 780 db_show_ccp_sc(struct ccp_softc *sc) 781 { 782 783 db_printf("ccp softc at %p\n", sc); 784 db_printf(" cid: %d\n", (int)sc->cid); 785 786 db_printf(" lock: "); 787 db_show_lock(&sc->lock); 788 789 db_printf(" detaching: %d\n", (int)sc->detaching); 790 db_printf(" ring_size_order: %u\n", sc->ring_size_order); 791 792 db_printf(" hw_version: %d\n", (int)sc->hw_version); 793 db_printf(" hw_features: %b\n", (int)sc->hw_features, 794 "\20\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA" 795 "\11SHA\0103DES\07AES"); 796 797 db_printf(" hw status:\n"); 798 db_ccp_show_hw(sc); 799 } 800 801 static void 802 db_show_ccp_qp(struct ccp_queue *qp) 803 { 804 805 db_printf(" lock: "); 806 db_show_lock(&qp->cq_lock); 807 808 db_printf(" cq_qindex: %u\n", qp->cq_qindex); 809 db_printf(" cq_softc: %p\n", qp->cq_softc); 810 811 db_printf(" head: %u\n", qp->cq_head); 812 db_printf(" tail: %u\n", qp->cq_tail); 813 db_printf(" acq_tail: %u\n", qp->cq_acq_tail); 814 db_printf(" desc_ring: %p\n", qp->desc_ring); 815 db_printf(" completions_ring: %p\n", qp->completions_ring); 816 db_printf(" descriptors (phys): 0x%jx\n", 817 (uintmax_t)qp->desc_ring_bus_addr); 818 819 db_printf(" hw status:\n"); 820 db_ccp_show_queue_hw(qp); 821 } 822 823 DB_SHOW_COMMAND(ccp, db_show_ccp) 824 { 825 struct ccp_softc *sc; 826 unsigned unit, qindex; 827 828 if (!have_addr) 829 goto usage; 830 831 unit = (unsigned)addr; 832 833 sc = devclass_get_softc(ccp_devclass, unit); 834 if (sc == NULL) { 835 db_printf("No such device ccp%u\n", unit); 836 goto usage; 837 } 838 839 if (count == -1) { 840 db_show_ccp_sc(sc); 841 return; 842 } 843 844 qindex = (unsigned)count; 845 if (qindex >= nitems(sc->queues)) { 846 db_printf("No such queue %u\n", qindex); 847 goto usage; 848 } 849 db_show_ccp_qp(&sc->queues[qindex]); 850 return; 851 852 usage: 853 db_printf("usage: show ccp <unit>[,<qindex>]\n"); 854 return; 855 } 856 #endif /* DDB */ 857