1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * This file contains the core framework routines for the 30 * kernel cryptographic framework. These routines are at the 31 * layer, between the kernel API/ioctls and the SPI. 32 */ 33 34 #include <sys/types.h> 35 #include <sys/errno.h> 36 #include <sys/kmem.h> 37 #include <sys/proc.h> 38 #include <sys/cpuvar.h> 39 #include <sys/cpupart.h> 40 #include <sys/ksynch.h> 41 #include <sys/callb.h> 42 #include <sys/cmn_err.h> 43 #include <sys/systm.h> 44 #include <sys/sysmacros.h> 45 #include <sys/kstat.h> 46 #include <sys/crypto/common.h> 47 #include <sys/crypto/impl.h> 48 #include <sys/crypto/sched_impl.h> 49 #include <sys/crypto/api.h> 50 #include <sys/crypto/spi.h> 51 #include <sys/taskq_impl.h> 52 #include <sys/ddi.h> 53 #include <sys/sunddi.h> 54 55 56 kcf_global_swq_t *gswq; /* Global software queue */ 57 58 /* Thread pool related variables */ 59 static kcf_pool_t *kcfpool; /* Thread pool of kcfd LWPs */ 60 int kcf_maxthreads; 61 int kcf_minthreads; 62 int kcf_thr_multiple = 2; /* Boot-time tunable for experimentation */ 63 static ulong_t kcf_idlethr_timeout; 64 static boolean_t kcf_sched_running = B_FALSE; 65 #define KCF_DEFAULT_THRTIMEOUT 60000000 /* 60 seconds */ 66 67 /* kmem caches used by the scheduler */ 68 static struct kmem_cache *kcf_sreq_cache; 69 static struct kmem_cache *kcf_areq_cache; 70 static struct kmem_cache *kcf_context_cache; 71 72 /* Global request ID table */ 73 static kcf_reqid_table_t *kcf_reqid_table[REQID_TABLES]; 74 75 /* KCF stats. Not protected. */ 76 static kcf_stats_t kcf_ksdata = { 77 { "total threads in pool", KSTAT_DATA_UINT32}, 78 { "idle threads in pool", KSTAT_DATA_UINT32}, 79 { "min threads in pool", KSTAT_DATA_UINT32}, 80 { "max threads in pool", KSTAT_DATA_UINT32}, 81 { "requests in gswq", KSTAT_DATA_UINT32}, 82 { "max requests in gswq", KSTAT_DATA_UINT32}, 83 { "minalloc for taskq", KSTAT_DATA_UINT32}, 84 { "maxalloc for taskq", KSTAT_DATA_UINT32} 85 }; 86 87 static kstat_t *kcf_misc_kstat = NULL; 88 ulong_t kcf_swprov_hndl = 0; 89 90 static kcf_areq_node_t *kcf_areqnode_alloc(kcf_provider_desc_t *, 91 kcf_context_t *, crypto_call_req_t *, kcf_req_params_t *, boolean_t); 92 static int kcf_disp_sw_request(kcf_areq_node_t *); 93 static void process_req_hwp(void *); 94 static kcf_areq_node_t *kcf_dequeue(); 95 static int kcf_enqueue(kcf_areq_node_t *); 96 static void kcf_failover_thread(); 97 static void kcfpool_alloc(); 98 static void kcf_reqid_delete(kcf_areq_node_t *areq); 99 static crypto_req_id_t kcf_reqid_insert(kcf_areq_node_t *areq); 100 static int kcf_misc_kstat_update(kstat_t *ksp, int rw); 101 static void compute_min_max_threads(); 102 103 104 /* 105 * Create a new context. 106 */ 107 crypto_ctx_t * 108 kcf_new_ctx(crypto_call_req_t *crq, kcf_provider_desc_t *pd, 109 crypto_session_id_t sid) 110 { 111 crypto_ctx_t *ctx; 112 kcf_context_t *kcf_ctx; 113 114 kcf_ctx = kmem_cache_alloc(kcf_context_cache, 115 (crq == NULL) ? KM_SLEEP : KM_NOSLEEP); 116 if (kcf_ctx == NULL) 117 return (NULL); 118 119 /* initialize the context for the consumer */ 120 kcf_ctx->kc_refcnt = 1; 121 kcf_ctx->kc_need_signal = B_FALSE; 122 kcf_ctx->kc_req_chain_first = NULL; 123 kcf_ctx->kc_req_chain_last = NULL; 124 kcf_ctx->kc_secondctx = NULL; 125 KCF_PROV_REFHOLD(pd); 126 kcf_ctx->kc_prov_desc = pd; 127 128 ctx = &kcf_ctx->kc_glbl_ctx; 129 ctx->cc_provider = pd->pd_prov_handle; 130 ctx->cc_session = sid; 131 ctx->cc_provider_private = NULL; 132 ctx->cc_framework_private = (void *)kcf_ctx; 133 134 return (ctx); 135 } 136 137 /* 138 * Allocate a new async request node. 139 * 140 * ictx - Framework private context pointer 141 * crq - Has callback function and argument. Should be non NULL. 142 * req - The parameters to pass to the SPI 143 */ 144 static kcf_areq_node_t * 145 kcf_areqnode_alloc(kcf_provider_desc_t *pd, kcf_context_t *ictx, 146 crypto_call_req_t *crq, kcf_req_params_t *req, boolean_t isdual) 147 { 148 kcf_areq_node_t *arptr, *areq; 149 150 ASSERT(crq != NULL); 151 arptr = kmem_cache_alloc(kcf_areq_cache, KM_NOSLEEP); 152 if (arptr == NULL) 153 return (NULL); 154 155 arptr->an_state = REQ_ALLOCATED; 156 arptr->an_reqarg = *crq; 157 arptr->an_params = *req; 158 arptr->an_context = ictx; 159 arptr->an_isdual = isdual; 160 161 arptr->an_next = arptr->an_prev = NULL; 162 KCF_PROV_REFHOLD(pd); 163 arptr->an_provider = pd; 164 arptr->an_tried_plist = NULL; 165 arptr->an_refcnt = 1; 166 arptr->an_idnext = arptr->an_idprev = NULL; 167 168 /* 169 * Requests for context-less operations do not use the 170 * fields - an_is_my_turn, and an_ctxchain_next. 171 */ 172 if (ictx == NULL) 173 return (arptr); 174 175 KCF_CONTEXT_REFHOLD(ictx); 176 /* 177 * Chain this request to the context. 178 */ 179 mutex_enter(&ictx->kc_in_use_lock); 180 arptr->an_ctxchain_next = NULL; 181 if ((areq = ictx->kc_req_chain_last) == NULL) { 182 arptr->an_is_my_turn = B_TRUE; 183 ictx->kc_req_chain_last = 184 ictx->kc_req_chain_first = arptr; 185 } else { 186 ASSERT(ictx->kc_req_chain_first != NULL); 187 arptr->an_is_my_turn = B_FALSE; 188 /* Insert the new request to the end of the chain. */ 189 areq->an_ctxchain_next = arptr; 190 ictx->kc_req_chain_last = arptr; 191 } 192 mutex_exit(&ictx->kc_in_use_lock); 193 194 return (arptr); 195 } 196 197 /* 198 * Queue the request node and do one of the following: 199 * - If there is an idle thread signal it to run. 200 * - If there is no idle thread and max running threads is not 201 * reached, signal the creator thread for more threads. 202 * 203 * If the two conditions above are not met, we don't need to do 204 * any thing. The request will be picked up by one of the 205 * worker threads when it becomes available. 206 */ 207 static int 208 kcf_disp_sw_request(kcf_areq_node_t *areq) 209 { 210 int err; 211 int cnt = 0; 212 213 if ((err = kcf_enqueue(areq)) != 0) 214 return (err); 215 216 if (kcfpool->kp_idlethreads > 0) { 217 /* Signal an idle thread to run */ 218 mutex_enter(&gswq->gs_lock); 219 cv_signal(&gswq->gs_cv); 220 mutex_exit(&gswq->gs_lock); 221 222 return (CRYPTO_QUEUED); 223 } 224 225 /* 226 * We keep the number of running threads to be at 227 * kcf_minthreads to reduce gs_lock contention. 228 */ 229 cnt = kcf_minthreads - 230 (kcfpool->kp_threads - kcfpool->kp_blockedthreads); 231 if (cnt > 0) { 232 /* 233 * The following ensures the number of threads in pool 234 * does not exceed kcf_maxthreads. 235 */ 236 cnt = min(cnt, kcf_maxthreads - kcfpool->kp_threads); 237 if (cnt > 0) { 238 /* Signal the creator thread for more threads */ 239 mutex_enter(&kcfpool->kp_user_lock); 240 if (!kcfpool->kp_signal_create_thread) { 241 kcfpool->kp_signal_create_thread = B_TRUE; 242 kcfpool->kp_nthrs = cnt; 243 cv_signal(&kcfpool->kp_user_cv); 244 } 245 mutex_exit(&kcfpool->kp_user_lock); 246 } 247 } 248 249 return (CRYPTO_QUEUED); 250 } 251 252 /* 253 * This routine is called by the taskq associated with 254 * each hardware provider. We notify the kernel consumer 255 * via the callback routine in case of CRYPTO_SUCCESS or 256 * a failure. 257 * 258 * A request can be of type kcf_areq_node_t or of type 259 * kcf_sreq_node_t. 260 */ 261 static void 262 process_req_hwp(void *ireq) 263 { 264 int error = 0; 265 crypto_ctx_t *ctx; 266 kcf_call_type_t ctype; 267 kcf_provider_desc_t *pd; 268 kcf_areq_node_t *areq = (kcf_areq_node_t *)ireq; 269 kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)ireq; 270 271 pd = ((ctype = GET_REQ_TYPE(ireq)) == CRYPTO_SYNCH) ? 272 sreq->sn_provider : areq->an_provider; 273 274 mutex_enter(&pd->pd_lock); 275 276 /* 277 * Wait if flow control is in effect for the provider. A 278 * CRYPTO_PROVIDER_READY or CRYPTO_PROVIDER_FAILED 279 * notification will signal us. We also get signaled if 280 * the provider is unregistering. 281 */ 282 while (pd->pd_state == KCF_PROV_BUSY) 283 cv_wait(&pd->pd_resume_cv, &pd->pd_lock); 284 285 /* 286 * Bump the internal reference count while the request is being 287 * processed. This is how we know when it's safe to unregister 288 * a provider. This step must precede the pd_state check below. 289 */ 290 KCF_PROV_IREFHOLD(pd); 291 292 /* 293 * Fail the request if the provider has failed. We return a 294 * recoverable error and the notified clients attempt any 295 * recovery. For async clients this is done in kcf_aop_done() 296 * and for sync clients it is done in the k-api routines. 297 */ 298 if (pd->pd_state >= KCF_PROV_FAILED) { 299 mutex_exit(&pd->pd_lock); 300 error = CRYPTO_DEVICE_ERROR; 301 goto bail; 302 } 303 304 mutex_exit(&pd->pd_lock); 305 306 if (ctype == CRYPTO_SYNCH) { 307 mutex_enter(&sreq->sn_lock); 308 sreq->sn_state = REQ_INPROGRESS; 309 mutex_exit(&sreq->sn_lock); 310 311 ctx = sreq->sn_context ? &sreq->sn_context->kc_glbl_ctx : NULL; 312 error = common_submit_request(sreq->sn_provider, ctx, 313 sreq->sn_params, sreq); 314 } else { 315 ASSERT(ctype == CRYPTO_ASYNCH); 316 317 mutex_enter(&areq->an_lock); 318 areq->an_state = REQ_INPROGRESS; 319 mutex_exit(&areq->an_lock); 320 321 /* 322 * We are in the per-hardware provider thread context and 323 * hence can sleep. Note that the caller would have done 324 * a taskq_dispatch(..., TQ_NOSLEEP) and would have returned. 325 */ 326 ctx = areq->an_context ? &areq->an_context->kc_glbl_ctx : NULL; 327 error = common_submit_request(areq->an_provider, ctx, 328 &areq->an_params, areq); 329 } 330 331 bail: 332 if (error == CRYPTO_QUEUED) { 333 /* 334 * The request is queued by the provider and we should 335 * get a crypto_op_notification() from the provider later. 336 * We notify the consumer at that time. 337 */ 338 return; 339 } else { /* CRYPTO_SUCCESS or other failure */ 340 KCF_PROV_IREFRELE(pd); 341 if (ctype == CRYPTO_SYNCH) 342 kcf_sop_done(sreq, error); 343 else 344 kcf_aop_done(areq, error); 345 } 346 } 347 348 /* 349 * This routine checks if a request can be retried on another 350 * provider. If true, mech1 is initialized to point to the mechanism 351 * structure. mech2 is also initialized in case of a dual operation. fg 352 * is initialized to the correct crypto_func_group_t bit flag. They are 353 * initialized by this routine, so that the caller can pass them to a 354 * kcf_get_mech_provider() or kcf_get_dual_provider() with no further change. 355 * 356 * We check that the request is for a init or atomic routine and that 357 * it is for one of the operation groups used from k-api . 358 */ 359 static boolean_t 360 can_resubmit(kcf_areq_node_t *areq, crypto_mechanism_t **mech1, 361 crypto_mechanism_t **mech2, crypto_func_group_t *fg) 362 { 363 kcf_req_params_t *params; 364 kcf_op_type_t optype; 365 366 params = &areq->an_params; 367 optype = params->rp_optype; 368 369 if (!(IS_INIT_OP(optype) || IS_ATOMIC_OP(optype))) 370 return (B_FALSE); 371 372 switch (params->rp_opgrp) { 373 case KCF_OG_DIGEST: { 374 kcf_digest_ops_params_t *dops = ¶ms->rp_u.digest_params; 375 376 dops->do_mech.cm_type = dops->do_framework_mechtype; 377 *mech1 = &dops->do_mech; 378 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_DIGEST : 379 CRYPTO_FG_DIGEST_ATOMIC; 380 break; 381 } 382 383 case KCF_OG_MAC: { 384 kcf_mac_ops_params_t *mops = ¶ms->rp_u.mac_params; 385 386 mops->mo_mech.cm_type = mops->mo_framework_mechtype; 387 *mech1 = &mops->mo_mech; 388 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_MAC : 389 CRYPTO_FG_MAC_ATOMIC; 390 break; 391 } 392 393 case KCF_OG_SIGN: { 394 kcf_sign_ops_params_t *sops = ¶ms->rp_u.sign_params; 395 396 sops->so_mech.cm_type = sops->so_framework_mechtype; 397 *mech1 = &sops->so_mech; 398 switch (optype) { 399 case KCF_OP_INIT: 400 *fg = CRYPTO_FG_SIGN; 401 break; 402 case KCF_OP_ATOMIC: 403 *fg = CRYPTO_FG_SIGN_ATOMIC; 404 break; 405 default: 406 ASSERT(optype == KCF_OP_SIGN_RECOVER_ATOMIC); 407 *fg = CRYPTO_FG_SIGN_RECOVER_ATOMIC; 408 } 409 break; 410 } 411 412 case KCF_OG_VERIFY: { 413 kcf_verify_ops_params_t *vops = ¶ms->rp_u.verify_params; 414 415 vops->vo_mech.cm_type = vops->vo_framework_mechtype; 416 *mech1 = &vops->vo_mech; 417 switch (optype) { 418 case KCF_OP_INIT: 419 *fg = CRYPTO_FG_VERIFY; 420 break; 421 case KCF_OP_ATOMIC: 422 *fg = CRYPTO_FG_VERIFY_ATOMIC; 423 break; 424 default: 425 ASSERT(optype == KCF_OP_VERIFY_RECOVER_ATOMIC); 426 *fg = CRYPTO_FG_VERIFY_RECOVER_ATOMIC; 427 } 428 break; 429 } 430 431 case KCF_OG_ENCRYPT: { 432 kcf_encrypt_ops_params_t *eops = ¶ms->rp_u.encrypt_params; 433 434 eops->eo_mech.cm_type = eops->eo_framework_mechtype; 435 *mech1 = &eops->eo_mech; 436 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_ENCRYPT : 437 CRYPTO_FG_ENCRYPT_ATOMIC; 438 break; 439 } 440 441 case KCF_OG_DECRYPT: { 442 kcf_decrypt_ops_params_t *dcrops = ¶ms->rp_u.decrypt_params; 443 444 dcrops->dop_mech.cm_type = dcrops->dop_framework_mechtype; 445 *mech1 = &dcrops->dop_mech; 446 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_DECRYPT : 447 CRYPTO_FG_DECRYPT_ATOMIC; 448 break; 449 } 450 451 case KCF_OG_ENCRYPT_MAC: { 452 kcf_encrypt_mac_ops_params_t *eops = 453 ¶ms->rp_u.encrypt_mac_params; 454 455 eops->em_encr_mech.cm_type = eops->em_framework_encr_mechtype; 456 *mech1 = &eops->em_encr_mech; 457 eops->em_mac_mech.cm_type = eops->em_framework_mac_mechtype; 458 *mech2 = &eops->em_mac_mech; 459 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_ENCRYPT_MAC : 460 CRYPTO_FG_ENCRYPT_MAC_ATOMIC; 461 break; 462 } 463 464 case KCF_OG_MAC_DECRYPT: { 465 kcf_mac_decrypt_ops_params_t *dops = 466 ¶ms->rp_u.mac_decrypt_params; 467 468 dops->md_mac_mech.cm_type = dops->md_framework_mac_mechtype; 469 *mech1 = &dops->md_mac_mech; 470 dops->md_decr_mech.cm_type = dops->md_framework_decr_mechtype; 471 *mech2 = &dops->md_decr_mech; 472 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_MAC_DECRYPT : 473 CRYPTO_FG_MAC_DECRYPT_ATOMIC; 474 break; 475 } 476 477 default: 478 return (B_FALSE); 479 } 480 481 return (B_TRUE); 482 } 483 484 /* 485 * This routine is called when a request to a provider has failed 486 * with a recoverable error. This routine tries to find another provider 487 * and dispatches the request to the new provider, if one is available. 488 * We reuse the request structure. 489 * 490 * A return value of NULL from kcf_get_mech_provider() indicates 491 * we have tried the last provider. 492 */ 493 static int 494 kcf_resubmit_request(kcf_areq_node_t *areq) 495 { 496 int error = CRYPTO_FAILED; 497 kcf_context_t *ictx; 498 kcf_provider_desc_t *old_pd; 499 kcf_provider_desc_t *new_pd; 500 crypto_mechanism_t *mech1 = NULL, *mech2 = NULL; 501 crypto_mech_type_t prov_mt1, prov_mt2; 502 crypto_func_group_t fg; 503 504 if (!can_resubmit(areq, &mech1, &mech2, &fg)) 505 return (error); 506 507 old_pd = areq->an_provider; 508 /* 509 * Add old_pd to the list of providers already tried. We release 510 * the hold on old_pd (from the earlier kcf_get_mech_provider()) in 511 * kcf_free_triedlist(). 512 */ 513 if (kcf_insert_triedlist(&areq->an_tried_plist, old_pd, 514 KM_NOSLEEP) == NULL) 515 return (error); 516 517 if (mech1 && !mech2) { 518 new_pd = kcf_get_mech_provider(mech1->cm_type, NULL, &error, 519 areq->an_tried_plist, fg, 520 (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), 0); 521 } else { 522 ASSERT(mech1 != NULL && mech2 != NULL); 523 524 new_pd = kcf_get_dual_provider(mech1, mech2, NULL, &prov_mt1, 525 &prov_mt2, &error, areq->an_tried_plist, fg, fg, 526 (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), 0); 527 } 528 529 if (new_pd == NULL) 530 return (error); 531 532 /* 533 * We reuse the old context by resetting provider specific 534 * fields in it. 535 */ 536 if ((ictx = areq->an_context) != NULL) { 537 crypto_ctx_t *ctx; 538 539 ASSERT(old_pd == ictx->kc_prov_desc); 540 KCF_PROV_REFRELE(ictx->kc_prov_desc); 541 KCF_PROV_REFHOLD(new_pd); 542 ictx->kc_prov_desc = new_pd; 543 544 ctx = &ictx->kc_glbl_ctx; 545 ctx->cc_provider = new_pd->pd_prov_handle; 546 ctx->cc_session = new_pd->pd_sid; 547 ctx->cc_provider_private = NULL; 548 } 549 550 /* We reuse areq. by resetting the provider and context fields. */ 551 KCF_PROV_REFRELE(old_pd); 552 KCF_PROV_REFHOLD(new_pd); 553 areq->an_provider = new_pd; 554 mutex_enter(&areq->an_lock); 555 areq->an_state = REQ_WAITING; 556 mutex_exit(&areq->an_lock); 557 558 switch (new_pd->pd_prov_type) { 559 case CRYPTO_SW_PROVIDER: 560 error = kcf_disp_sw_request(areq); 561 break; 562 563 case CRYPTO_HW_PROVIDER: { 564 taskq_t *taskq = new_pd->pd_sched_info.ks_taskq; 565 566 if (taskq_dispatch(taskq, process_req_hwp, areq, TQ_NOSLEEP) == 567 (taskqid_t)0) { 568 error = CRYPTO_HOST_MEMORY; 569 } else { 570 error = CRYPTO_QUEUED; 571 } 572 573 break; 574 } 575 } 576 577 return (error); 578 } 579 580 #define EMPTY_TASKQ(tq) ((tq)->tq_task.tqent_next == &(tq)->tq_task) 581 582 /* 583 * Routine called by both ioctl and k-api. The consumer should 584 * bundle the parameters into a kcf_req_params_t structure. A bunch 585 * of macros are available in ops_impl.h for this bundling. They are: 586 * 587 * KCF_WRAP_DIGEST_OPS_PARAMS() 588 * KCF_WRAP_MAC_OPS_PARAMS() 589 * KCF_WRAP_ENCRYPT_OPS_PARAMS() 590 * KCF_WRAP_DECRYPT_OPS_PARAMS() ... etc. 591 * 592 * It is the caller's responsibility to free the ctx argument when 593 * appropriate. See the KCF_CONTEXT_COND_RELEASE macro for details. 594 */ 595 int 596 kcf_submit_request(kcf_provider_desc_t *pd, crypto_ctx_t *ctx, 597 crypto_call_req_t *crq, kcf_req_params_t *params, boolean_t cont) 598 { 599 int error = CRYPTO_SUCCESS; 600 kcf_areq_node_t *areq; 601 kcf_sreq_node_t *sreq; 602 kcf_context_t *kcf_ctx; 603 taskq_t *taskq = pd->pd_sched_info.ks_taskq; 604 605 kcf_ctx = ctx ? (kcf_context_t *)ctx->cc_framework_private : NULL; 606 607 /* Synchronous cases */ 608 if (crq == NULL) { 609 switch (pd->pd_prov_type) { 610 case CRYPTO_SW_PROVIDER: 611 error = common_submit_request(pd, ctx, params, 612 KCF_RHNDL(KM_SLEEP)); 613 break; 614 615 case CRYPTO_HW_PROVIDER: 616 sreq = kmem_cache_alloc(kcf_sreq_cache, KM_SLEEP); 617 sreq->sn_state = REQ_ALLOCATED; 618 sreq->sn_rv = CRYPTO_FAILED; 619 620 sreq->sn_params = params; 621 KCF_PROV_REFHOLD(pd); 622 sreq->sn_provider = pd; 623 624 /* 625 * Note that we do not need to hold the context 626 * for synchronous case as the context will never 627 * become invalid underneath us in this case. 628 */ 629 sreq->sn_context = kcf_ctx; 630 631 ASSERT(taskq != NULL); 632 /* 633 * Call the SPI directly if the taskq is empty and the 634 * provider is not busy, else dispatch to the taskq. 635 * Calling directly is fine as this is the synchronous 636 * case. This is unlike the asynchronous case where we 637 * must always dispatch to the taskq. 638 */ 639 if (EMPTY_TASKQ(taskq) && 640 pd->pd_state == KCF_PROV_READY) { 641 process_req_hwp(sreq); 642 } else { 643 /* 644 * We can not tell from taskq_dispatch() return 645 * value if we exceeded maxalloc. Hence the 646 * check here. Since we are allowed to wait in 647 * the synchronous case, we wait for the taskq 648 * to become empty. 649 */ 650 if (taskq->tq_nalloc >= crypto_taskq_maxalloc) { 651 taskq_wait(taskq); 652 } 653 if (taskq_dispatch(taskq, process_req_hwp, 654 sreq, TQ_SLEEP) == (taskqid_t)0) { 655 error = CRYPTO_HOST_MEMORY; 656 KCF_PROV_REFRELE(sreq->sn_provider); 657 kmem_cache_free(kcf_sreq_cache, sreq); 658 goto done; 659 } 660 } 661 662 /* 663 * Wait for the notification to arrive, 664 * if the operation is not done yet. 665 * Bug# 4722589 will make the wait a cv_wait_sig(). 666 */ 667 mutex_enter(&sreq->sn_lock); 668 while (sreq->sn_state < REQ_DONE) 669 cv_wait(&sreq->sn_cv, &sreq->sn_lock); 670 mutex_exit(&sreq->sn_lock); 671 672 error = sreq->sn_rv; 673 KCF_PROV_REFRELE(sreq->sn_provider); 674 kmem_cache_free(kcf_sreq_cache, sreq); 675 676 break; 677 678 default: 679 error = CRYPTO_FAILED; 680 break; 681 } 682 683 } else { /* Asynchronous cases */ 684 switch (pd->pd_prov_type) { 685 case CRYPTO_SW_PROVIDER: 686 if (!(crq->cr_flag & CRYPTO_ALWAYS_QUEUE)) { 687 /* 688 * This case has less overhead since there is 689 * no switching of context. 690 */ 691 error = common_submit_request(pd, ctx, params, 692 KCF_RHNDL(KM_NOSLEEP)); 693 } else { 694 /* 695 * CRYPTO_ALWAYS_QUEUE is set. We need to 696 * queue the request and return. 697 */ 698 areq = kcf_areqnode_alloc(pd, kcf_ctx, crq, 699 params, cont); 700 if (areq == NULL) 701 error = CRYPTO_HOST_MEMORY; 702 else { 703 if (!(crq->cr_flag 704 & CRYPTO_SKIP_REQID)) { 705 /* 706 * Set the request handle. This handle 707 * is used for any crypto_cancel_req(9f) 708 * calls from the consumer. We have to 709 * do this before dispatching the 710 * request. 711 */ 712 crq->cr_reqid = kcf_reqid_insert(areq); 713 } 714 715 error = kcf_disp_sw_request(areq); 716 /* 717 * There is an error processing this 718 * request. Remove the handle and 719 * release the request structure. 720 */ 721 if (error != CRYPTO_QUEUED) { 722 if (!(crq->cr_flag 723 & CRYPTO_SKIP_REQID)) 724 kcf_reqid_delete(areq); 725 KCF_AREQ_REFRELE(areq); 726 } 727 } 728 } 729 break; 730 731 case CRYPTO_HW_PROVIDER: 732 /* 733 * We need to queue the request and return. 734 */ 735 areq = kcf_areqnode_alloc(pd, kcf_ctx, crq, params, 736 cont); 737 if (areq == NULL) { 738 error = CRYPTO_HOST_MEMORY; 739 goto done; 740 } 741 742 ASSERT(taskq != NULL); 743 /* 744 * We can not tell from taskq_dispatch() return 745 * value if we exceeded maxalloc. Hence the check 746 * here. 747 */ 748 if (taskq->tq_nalloc >= crypto_taskq_maxalloc) { 749 error = CRYPTO_BUSY; 750 KCF_AREQ_REFRELE(areq); 751 goto done; 752 } 753 754 if (!(crq->cr_flag & CRYPTO_SKIP_REQID)) { 755 /* 756 * Set the request handle. This handle is used 757 * for any crypto_cancel_req(9f) calls from the 758 * consumer. We have to do this before dispatching 759 * the request. 760 */ 761 crq->cr_reqid = kcf_reqid_insert(areq); 762 } 763 764 if (taskq_dispatch(taskq, 765 process_req_hwp, areq, TQ_NOSLEEP) == 766 (taskqid_t)0) { 767 error = CRYPTO_HOST_MEMORY; 768 if (!(crq->cr_flag & CRYPTO_SKIP_REQID)) 769 kcf_reqid_delete(areq); 770 KCF_AREQ_REFRELE(areq); 771 } else { 772 error = CRYPTO_QUEUED; 773 } 774 break; 775 776 default: 777 error = CRYPTO_FAILED; 778 break; 779 } 780 } 781 782 done: 783 return (error); 784 } 785 786 /* 787 * We're done with this framework context, so free it. Note that freeing 788 * framework context (kcf_context) frees the global context (crypto_ctx). 789 * 790 * The provider is responsible for freeing provider private context after a 791 * final or single operation and resetting the cc_provider_private field 792 * to NULL. It should do this before it notifies the framework of the 793 * completion. We still need to call KCF_PROV_FREE_CONTEXT to handle cases 794 * like crypto_cancel_ctx(9f). 795 */ 796 void 797 kcf_free_context(kcf_context_t *kcf_ctx) 798 { 799 kcf_provider_desc_t *pd = kcf_ctx->kc_prov_desc; 800 crypto_ctx_t *gctx = &kcf_ctx->kc_glbl_ctx; 801 kcf_context_t *kcf_secondctx = kcf_ctx->kc_secondctx; 802 803 /* Release the second context, if any */ 804 805 if (kcf_secondctx != NULL) 806 KCF_CONTEXT_REFRELE(kcf_secondctx); 807 808 if (gctx->cc_provider_private != NULL) { 809 mutex_enter(&pd->pd_lock); 810 if (!KCF_IS_PROV_REMOVED(pd)) { 811 /* 812 * Increment the provider's internal refcnt so it 813 * doesn't unregister from the framework while 814 * we're calling the entry point. 815 */ 816 KCF_PROV_IREFHOLD(pd); 817 mutex_exit(&pd->pd_lock); 818 (void) KCF_PROV_FREE_CONTEXT(pd, gctx); 819 KCF_PROV_IREFRELE(pd); 820 } else { 821 mutex_exit(&pd->pd_lock); 822 } 823 } 824 825 /* kcf_ctx->kc_prov_desc has a hold on pd */ 826 KCF_PROV_REFRELE(kcf_ctx->kc_prov_desc); 827 828 kmem_cache_free(kcf_context_cache, kcf_ctx); 829 } 830 831 /* 832 * Free the request after releasing all the holds. 833 */ 834 void 835 kcf_free_req(kcf_areq_node_t *areq) 836 { 837 KCF_PROV_REFRELE(areq->an_provider); 838 if (areq->an_context != NULL) 839 KCF_CONTEXT_REFRELE(areq->an_context); 840 841 if (areq->an_tried_plist != NULL) 842 kcf_free_triedlist(areq->an_tried_plist); 843 kmem_cache_free(kcf_areq_cache, areq); 844 } 845 846 /* 847 * Utility routine to remove a request from the chain of requests 848 * hanging off a context. 849 */ 850 void 851 kcf_removereq_in_ctxchain(kcf_context_t *ictx, kcf_areq_node_t *areq) 852 { 853 kcf_areq_node_t *cur, *prev; 854 855 /* 856 * Get context lock, search for areq in the chain and remove it. 857 */ 858 ASSERT(ictx != NULL); 859 mutex_enter(&ictx->kc_in_use_lock); 860 prev = cur = ictx->kc_req_chain_first; 861 862 while (cur != NULL) { 863 if (cur == areq) { 864 if (prev == cur) { 865 if ((ictx->kc_req_chain_first = 866 cur->an_ctxchain_next) == NULL) 867 ictx->kc_req_chain_last = NULL; 868 } else { 869 if (cur == ictx->kc_req_chain_last) 870 ictx->kc_req_chain_last = prev; 871 prev->an_ctxchain_next = cur->an_ctxchain_next; 872 } 873 874 break; 875 } 876 prev = cur; 877 cur = cur->an_ctxchain_next; 878 } 879 mutex_exit(&ictx->kc_in_use_lock); 880 } 881 882 /* 883 * Remove the specified node from the global software queue. 884 * 885 * The caller must hold the queue lock and request lock (an_lock). 886 */ 887 void 888 kcf_remove_node(kcf_areq_node_t *node) 889 { 890 kcf_areq_node_t *nextp = node->an_next; 891 kcf_areq_node_t *prevp = node->an_prev; 892 893 ASSERT(mutex_owned(&gswq->gs_lock)); 894 895 if (nextp != NULL) 896 nextp->an_prev = prevp; 897 else 898 gswq->gs_last = prevp; 899 900 if (prevp != NULL) 901 prevp->an_next = nextp; 902 else 903 gswq->gs_first = nextp; 904 905 ASSERT(mutex_owned(&node->an_lock)); 906 node->an_state = REQ_CANCELED; 907 } 908 909 /* 910 * Remove and return the first node in the global software queue. 911 * 912 * The caller must hold the queue lock. 913 */ 914 static kcf_areq_node_t * 915 kcf_dequeue() 916 { 917 kcf_areq_node_t *tnode = NULL; 918 919 ASSERT(mutex_owned(&gswq->gs_lock)); 920 if ((tnode = gswq->gs_first) == NULL) { 921 return (NULL); 922 } else { 923 ASSERT(gswq->gs_first->an_prev == NULL); 924 gswq->gs_first = tnode->an_next; 925 if (tnode->an_next == NULL) 926 gswq->gs_last = NULL; 927 else 928 tnode->an_next->an_prev = NULL; 929 } 930 931 gswq->gs_njobs--; 932 return (tnode); 933 } 934 935 /* 936 * Add the request node to the end of the global software queue. 937 * 938 * The caller should not hold the queue lock. Returns 0 if the 939 * request is successfully queued. Returns CRYPTO_BUSY if the limit 940 * on the number of jobs is exceeded. 941 */ 942 static int 943 kcf_enqueue(kcf_areq_node_t *node) 944 { 945 kcf_areq_node_t *tnode; 946 947 mutex_enter(&gswq->gs_lock); 948 949 if (gswq->gs_njobs >= gswq->gs_maxjobs) { 950 mutex_exit(&gswq->gs_lock); 951 return (CRYPTO_BUSY); 952 } 953 954 if (gswq->gs_last == NULL) { 955 gswq->gs_first = gswq->gs_last = node; 956 } else { 957 ASSERT(gswq->gs_last->an_next == NULL); 958 tnode = gswq->gs_last; 959 tnode->an_next = node; 960 gswq->gs_last = node; 961 node->an_prev = tnode; 962 } 963 964 gswq->gs_njobs++; 965 966 /* an_lock not needed here as we hold gs_lock */ 967 node->an_state = REQ_WAITING; 968 969 mutex_exit(&gswq->gs_lock); 970 971 return (0); 972 } 973 974 /* 975 * Decrement the thread pool count and signal the failover 976 * thread if we are the last one out. 977 */ 978 static void 979 kcf_decrcnt_andsignal() 980 { 981 KCF_ATOMIC_DECR(kcfpool->kp_threads); 982 983 mutex_enter(&kcfpool->kp_thread_lock); 984 if (kcfpool->kp_threads == 0) 985 cv_signal(&kcfpool->kp_nothr_cv); 986 mutex_exit(&kcfpool->kp_thread_lock); 987 } 988 989 /* 990 * Function run by a thread from kcfpool to work on global software queue. 991 * It is called from ioctl(CRYPTO_POOL_RUN, ...). 992 */ 993 int 994 kcf_svc_do_run(void) 995 { 996 int error = 0; 997 clock_t rv; 998 clock_t timeout_val; 999 kcf_areq_node_t *req; 1000 kcf_context_t *ictx; 1001 kcf_provider_desc_t *pd; 1002 1003 KCF_ATOMIC_INCR(kcfpool->kp_threads); 1004 1005 for (;;) { 1006 mutex_enter(&gswq->gs_lock); 1007 1008 while ((req = kcf_dequeue()) == NULL) { 1009 timeout_val = ddi_get_lbolt() + 1010 drv_usectohz(kcf_idlethr_timeout); 1011 1012 KCF_ATOMIC_INCR(kcfpool->kp_idlethreads); 1013 rv = cv_timedwait_sig(&gswq->gs_cv, &gswq->gs_lock, 1014 timeout_val); 1015 KCF_ATOMIC_DECR(kcfpool->kp_idlethreads); 1016 1017 switch (rv) { 1018 case 0: 1019 /* 1020 * A signal (as in kill(2)) is pending. We did 1021 * not get any cv_signal(). 1022 */ 1023 kcf_decrcnt_andsignal(); 1024 mutex_exit(&gswq->gs_lock); 1025 return (EINTR); 1026 1027 case -1: 1028 /* 1029 * Timed out and we are not signaled. Let us 1030 * see if this thread should exit. We should 1031 * keep at least kcf_minthreads. 1032 */ 1033 if (kcfpool->kp_threads > kcf_minthreads) { 1034 kcf_decrcnt_andsignal(); 1035 mutex_exit(&gswq->gs_lock); 1036 return (0); 1037 } 1038 1039 /* Resume the wait for work */ 1040 break; 1041 1042 default: 1043 /* 1044 * We are signaled to work on the queue. 1045 */ 1046 break; 1047 } 1048 } 1049 1050 mutex_exit(&gswq->gs_lock); 1051 1052 ictx = req->an_context; 1053 if (ictx == NULL) { /* Context-less operation */ 1054 pd = req->an_provider; 1055 error = common_submit_request(pd, NULL, 1056 &req->an_params, req); 1057 kcf_aop_done(req, error); 1058 continue; 1059 } 1060 1061 /* 1062 * We check if we can work on the request now. 1063 * Solaris does not guarantee any order on how the threads 1064 * are scheduled or how the waiters on a mutex are chosen. 1065 * So, we need to maintain our own order. 1066 * 1067 * is_my_turn is set to B_TRUE initially for a request when 1068 * it is enqueued and there are no other requests 1069 * for that context. Note that a thread sleeping on 1070 * kc_in_use_cv is not counted as an idle thread. This is 1071 * because we define an idle thread as one that sleeps on the 1072 * global queue waiting for new requests. 1073 */ 1074 mutex_enter(&ictx->kc_in_use_lock); 1075 while (req->an_is_my_turn == B_FALSE) { 1076 ictx->kc_need_signal = B_TRUE; 1077 KCF_ATOMIC_INCR(kcfpool->kp_blockedthreads); 1078 cv_wait(&ictx->kc_in_use_cv, &ictx->kc_in_use_lock); 1079 KCF_ATOMIC_DECR(kcfpool->kp_blockedthreads); 1080 } 1081 mutex_exit(&ictx->kc_in_use_lock); 1082 1083 mutex_enter(&req->an_lock); 1084 req->an_state = REQ_INPROGRESS; 1085 mutex_exit(&req->an_lock); 1086 1087 pd = ictx->kc_prov_desc; 1088 ASSERT(pd == req->an_provider); 1089 error = common_submit_request(pd, &ictx->kc_glbl_ctx, 1090 &req->an_params, req); 1091 1092 kcf_aop_done(req, error); 1093 } 1094 } 1095 1096 /* 1097 * kmem_cache_alloc constructor for sync request structure. 1098 */ 1099 /* ARGSUSED */ 1100 static int 1101 kcf_sreq_cache_constructor(void *buf, void *cdrarg, int kmflags) 1102 { 1103 kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)buf; 1104 1105 sreq->sn_type = CRYPTO_SYNCH; 1106 cv_init(&sreq->sn_cv, NULL, CV_DEFAULT, NULL); 1107 mutex_init(&sreq->sn_lock, NULL, MUTEX_DEFAULT, NULL); 1108 1109 return (0); 1110 } 1111 1112 /* ARGSUSED */ 1113 static void 1114 kcf_sreq_cache_destructor(void *buf, void *cdrarg) 1115 { 1116 kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)buf; 1117 1118 mutex_destroy(&sreq->sn_lock); 1119 cv_destroy(&sreq->sn_cv); 1120 } 1121 1122 /* 1123 * kmem_cache_alloc constructor for async request structure. 1124 */ 1125 /* ARGSUSED */ 1126 static int 1127 kcf_areq_cache_constructor(void *buf, void *cdrarg, int kmflags) 1128 { 1129 kcf_areq_node_t *areq = (kcf_areq_node_t *)buf; 1130 1131 areq->an_type = CRYPTO_ASYNCH; 1132 mutex_init(&areq->an_lock, NULL, MUTEX_DEFAULT, NULL); 1133 cv_init(&areq->an_done, NULL, CV_DEFAULT, NULL); 1134 1135 return (0); 1136 } 1137 1138 /* ARGSUSED */ 1139 static void 1140 kcf_areq_cache_destructor(void *buf, void *cdrarg) 1141 { 1142 kcf_areq_node_t *areq = (kcf_areq_node_t *)buf; 1143 1144 ASSERT(areq->an_refcnt == 0); 1145 mutex_destroy(&areq->an_lock); 1146 cv_destroy(&areq->an_done); 1147 } 1148 1149 /* 1150 * kmem_cache_alloc constructor for kcf_context structure. 1151 */ 1152 /* ARGSUSED */ 1153 static int 1154 kcf_context_cache_constructor(void *buf, void *cdrarg, int kmflags) 1155 { 1156 kcf_context_t *kctx = (kcf_context_t *)buf; 1157 1158 mutex_init(&kctx->kc_in_use_lock, NULL, MUTEX_DEFAULT, NULL); 1159 cv_init(&kctx->kc_in_use_cv, NULL, CV_DEFAULT, NULL); 1160 1161 return (0); 1162 } 1163 1164 /* ARGSUSED */ 1165 static void 1166 kcf_context_cache_destructor(void *buf, void *cdrarg) 1167 { 1168 kcf_context_t *kctx = (kcf_context_t *)buf; 1169 1170 ASSERT(kctx->kc_refcnt == 0); 1171 mutex_destroy(&kctx->kc_in_use_lock); 1172 cv_destroy(&kctx->kc_in_use_cv); 1173 } 1174 1175 /* 1176 * Creates and initializes all the structures needed by the framework. 1177 */ 1178 void 1179 kcf_sched_init(void) 1180 { 1181 int i; 1182 kcf_reqid_table_t *rt; 1183 1184 /* 1185 * Create all the kmem caches needed by the framework. We set the 1186 * align argument to 64, to get a slab aligned to 64-byte as well as 1187 * have the objects (cache_chunksize) to be a 64-byte multiple. 1188 * This helps to avoid false sharing as this is the size of the 1189 * CPU cache line. 1190 */ 1191 kcf_sreq_cache = kmem_cache_create("kcf_sreq_cache", 1192 sizeof (struct kcf_sreq_node), 64, kcf_sreq_cache_constructor, 1193 kcf_sreq_cache_destructor, NULL, NULL, NULL, 0); 1194 1195 kcf_areq_cache = kmem_cache_create("kcf_areq_cache", 1196 sizeof (struct kcf_areq_node), 64, kcf_areq_cache_constructor, 1197 kcf_areq_cache_destructor, NULL, NULL, NULL, 0); 1198 1199 kcf_context_cache = kmem_cache_create("kcf_context_cache", 1200 sizeof (struct kcf_context), 64, kcf_context_cache_constructor, 1201 kcf_context_cache_destructor, NULL, NULL, NULL, 0); 1202 1203 mutex_init(&kcf_dh_lock, NULL, MUTEX_DEFAULT, NULL); 1204 1205 gswq = kmem_alloc(sizeof (kcf_global_swq_t), KM_SLEEP); 1206 1207 mutex_init(&gswq->gs_lock, NULL, MUTEX_DEFAULT, NULL); 1208 cv_init(&gswq->gs_cv, NULL, CV_DEFAULT, NULL); 1209 gswq->gs_njobs = 0; 1210 compute_min_max_threads(); /* Computes gs_maxjobs also. */ 1211 gswq->gs_first = gswq->gs_last = NULL; 1212 1213 /* Initialize the global reqid table */ 1214 for (i = 0; i < REQID_TABLES; i++) { 1215 rt = kmem_zalloc(sizeof (kcf_reqid_table_t), KM_SLEEP); 1216 kcf_reqid_table[i] = rt; 1217 mutex_init(&rt->rt_lock, NULL, MUTEX_DEFAULT, NULL); 1218 rt->rt_curid = i; 1219 } 1220 1221 /* Allocate and initialize the thread pool */ 1222 kcfpool_alloc(); 1223 1224 /* Initialize the event notification list variables */ 1225 mutex_init(&ntfy_list_lock, NULL, MUTEX_DEFAULT, NULL); 1226 cv_init(&ntfy_list_cv, NULL, CV_DEFAULT, NULL); 1227 1228 /* Initialize the crypto_bufcall list variables */ 1229 mutex_init(&cbuf_list_lock, NULL, MUTEX_DEFAULT, NULL); 1230 cv_init(&cbuf_list_cv, NULL, CV_DEFAULT, NULL); 1231 1232 /* Create the kcf kstat */ 1233 kcf_misc_kstat = kstat_create("kcf", 0, "framework_stats", "crypto", 1234 KSTAT_TYPE_NAMED, sizeof (kcf_stats_t) / sizeof (kstat_named_t), 1235 KSTAT_FLAG_VIRTUAL); 1236 1237 if (kcf_misc_kstat != NULL) { 1238 kcf_misc_kstat->ks_data = &kcf_ksdata; 1239 kcf_misc_kstat->ks_update = kcf_misc_kstat_update; 1240 kstat_install(kcf_misc_kstat); 1241 } 1242 } 1243 1244 /* 1245 * This routine should only be called by drv/cryptoadm. 1246 * 1247 * kcf_sched_running flag isn't protected by a lock. But, we are safe because 1248 * the first thread ("cryptoadm refresh") calling this routine during 1249 * boot time completes before any other thread that can call this routine. 1250 */ 1251 void 1252 kcf_sched_start(void) 1253 { 1254 if (kcf_sched_running) 1255 return; 1256 1257 /* Start the failover kernel thread for now */ 1258 (void) thread_create(NULL, 0, &kcf_failover_thread, 0, 0, &p0, 1259 TS_RUN, minclsyspri); 1260 1261 /* Start the background processing thread. */ 1262 (void) thread_create(NULL, 0, &crypto_bufcall_service, 0, 0, &p0, 1263 TS_RUN, minclsyspri); 1264 1265 kcf_sched_running = B_TRUE; 1266 } 1267 1268 /* 1269 * Signal the waiting sync client. 1270 */ 1271 void 1272 kcf_sop_done(kcf_sreq_node_t *sreq, int error) 1273 { 1274 mutex_enter(&sreq->sn_lock); 1275 sreq->sn_state = REQ_DONE; 1276 sreq->sn_rv = error; 1277 cv_signal(&sreq->sn_cv); 1278 mutex_exit(&sreq->sn_lock); 1279 } 1280 1281 /* 1282 * Callback the async client with the operation status. 1283 * We free the async request node and possibly the context. 1284 * We also handle any chain of requests hanging off of 1285 * the context. 1286 */ 1287 void 1288 kcf_aop_done(kcf_areq_node_t *areq, int error) 1289 { 1290 kcf_op_type_t optype; 1291 boolean_t skip_notify = B_FALSE; 1292 kcf_context_t *ictx; 1293 kcf_areq_node_t *nextreq; 1294 1295 /* 1296 * Handle recoverable errors. This has to be done first 1297 * before doing any thing else in this routine so that 1298 * we do not change the state of the request. 1299 */ 1300 if (error != CRYPTO_SUCCESS && IS_RECOVERABLE(error)) { 1301 /* 1302 * We try another provider, if one is available. Else 1303 * we continue with the failure notification to the 1304 * client. 1305 */ 1306 if (kcf_resubmit_request(areq) == CRYPTO_QUEUED) 1307 return; 1308 } 1309 1310 mutex_enter(&areq->an_lock); 1311 areq->an_state = REQ_DONE; 1312 mutex_exit(&areq->an_lock); 1313 1314 optype = (&areq->an_params)->rp_optype; 1315 if ((ictx = areq->an_context) != NULL) { 1316 /* 1317 * A request after it is removed from the request 1318 * queue, still stays on a chain of requests hanging 1319 * of its context structure. It needs to be removed 1320 * from this chain at this point. 1321 */ 1322 mutex_enter(&ictx->kc_in_use_lock); 1323 nextreq = areq->an_ctxchain_next; 1324 ASSERT(nextreq != NULL || ictx->kc_need_signal == B_FALSE); 1325 1326 if (nextreq != NULL) { 1327 nextreq->an_is_my_turn = B_TRUE; 1328 /* 1329 * Currently, the following case happens 1330 * only for software providers. 1331 */ 1332 if (ictx->kc_need_signal) { 1333 cv_broadcast(&ictx->kc_in_use_cv); 1334 ictx->kc_need_signal = B_FALSE; 1335 } 1336 } 1337 1338 ictx->kc_req_chain_first = nextreq; 1339 if (nextreq == NULL) 1340 ictx->kc_req_chain_last = NULL; 1341 mutex_exit(&ictx->kc_in_use_lock); 1342 1343 if (IS_SINGLE_OP(optype) || IS_FINAL_OP(optype)) { 1344 ASSERT(nextreq == NULL); 1345 KCF_CONTEXT_REFRELE(ictx); 1346 } else if (error != CRYPTO_SUCCESS && IS_INIT_OP(optype)) { 1347 /* 1348 * NOTE - We do not release the context in case of update 1349 * operations. We require the consumer to free it explicitly, 1350 * in case it wants to abandon an update operation. This is done 1351 * as there may be mechanisms in ECB mode that can continue 1352 * even if an operation on a block fails. 1353 */ 1354 KCF_CONTEXT_REFRELE(ictx); 1355 } 1356 } 1357 1358 /* Deal with the internal continuation to this request first */ 1359 1360 if (areq->an_isdual) { 1361 kcf_dual_req_t *next_arg; 1362 next_arg = (kcf_dual_req_t *)areq->an_reqarg.cr_callback_arg; 1363 next_arg->kr_areq = areq; 1364 KCF_AREQ_REFHOLD(areq); 1365 areq->an_isdual = B_FALSE; 1366 1367 NOTIFY_CLIENT(areq, error); 1368 return; 1369 } 1370 1371 /* 1372 * If CRYPTO_NOTIFY_OPDONE flag is set, we should notify 1373 * always. If this flag is clear, we skip the notification 1374 * provided there are no errors. We check this flag for only 1375 * init or update operations. It is ignored for single, final or 1376 * atomic operations. 1377 */ 1378 skip_notify = (IS_UPDATE_OP(optype) || IS_INIT_OP(optype)) && 1379 (!(areq->an_reqarg.cr_flag & CRYPTO_NOTIFY_OPDONE)) && 1380 (error == CRYPTO_SUCCESS); 1381 1382 if (!skip_notify) { 1383 NOTIFY_CLIENT(areq, error); 1384 } 1385 1386 if (!(areq->an_reqarg.cr_flag & CRYPTO_SKIP_REQID)) 1387 kcf_reqid_delete(areq); 1388 1389 KCF_AREQ_REFRELE(areq); 1390 } 1391 1392 /* 1393 * Allocate the thread pool and initialize all the fields. 1394 */ 1395 static void 1396 kcfpool_alloc() 1397 { 1398 kcfpool = kmem_alloc(sizeof (kcf_pool_t), KM_SLEEP); 1399 1400 kcfpool->kp_threads = kcfpool->kp_idlethreads = 0; 1401 kcfpool->kp_blockedthreads = 0; 1402 kcfpool->kp_signal_create_thread = B_FALSE; 1403 kcfpool->kp_nthrs = 0; 1404 kcfpool->kp_user_waiting = B_FALSE; 1405 1406 mutex_init(&kcfpool->kp_thread_lock, NULL, MUTEX_DEFAULT, NULL); 1407 cv_init(&kcfpool->kp_nothr_cv, NULL, CV_DEFAULT, NULL); 1408 1409 mutex_init(&kcfpool->kp_user_lock, NULL, MUTEX_DEFAULT, NULL); 1410 cv_init(&kcfpool->kp_user_cv, NULL, CV_DEFAULT, NULL); 1411 1412 kcf_idlethr_timeout = KCF_DEFAULT_THRTIMEOUT; 1413 } 1414 1415 /* 1416 * This function is run by the 'creator' thread in the pool. 1417 * It is called from ioctl(CRYPTO_POOL_WAIT, ...). 1418 */ 1419 int 1420 kcf_svc_wait(int *nthrs) 1421 { 1422 clock_t rv; 1423 clock_t timeout_val; 1424 1425 if (kcfpool == NULL) 1426 return (ENOENT); 1427 1428 mutex_enter(&kcfpool->kp_user_lock); 1429 /* Check if there's already a user thread waiting on this kcfpool */ 1430 if (kcfpool->kp_user_waiting) { 1431 mutex_exit(&kcfpool->kp_user_lock); 1432 *nthrs = 0; 1433 return (EBUSY); 1434 } 1435 1436 kcfpool->kp_user_waiting = B_TRUE; 1437 1438 /* Go to sleep, waiting for the signaled flag. */ 1439 while (!kcfpool->kp_signal_create_thread) { 1440 timeout_val = ddi_get_lbolt() + 1441 drv_usectohz(kcf_idlethr_timeout); 1442 1443 rv = cv_timedwait_sig(&kcfpool->kp_user_cv, 1444 &kcfpool->kp_user_lock, timeout_val); 1445 switch (rv) { 1446 case 0: 1447 /* Interrupted, return to handle exit or signal */ 1448 kcfpool->kp_user_waiting = B_FALSE; 1449 kcfpool->kp_signal_create_thread = B_FALSE; 1450 mutex_exit(&kcfpool->kp_user_lock); 1451 /* 1452 * kcfd is exiting. Release the door and 1453 * invalidate it. 1454 */ 1455 mutex_enter(&kcf_dh_lock); 1456 if (kcf_dh != NULL) { 1457 door_ki_rele(kcf_dh); 1458 kcf_dh = NULL; 1459 } 1460 mutex_exit(&kcf_dh_lock); 1461 return (EINTR); 1462 1463 case -1: 1464 /* Timed out. Recalculate the min/max threads */ 1465 compute_min_max_threads(); 1466 break; 1467 1468 default: 1469 /* Worker thread did a cv_signal() */ 1470 break; 1471 } 1472 } 1473 1474 kcfpool->kp_signal_create_thread = B_FALSE; 1475 kcfpool->kp_user_waiting = B_FALSE; 1476 1477 *nthrs = kcfpool->kp_nthrs; 1478 mutex_exit(&kcfpool->kp_user_lock); 1479 1480 /* Return to userland for possible thread creation. */ 1481 return (0); 1482 } 1483 1484 1485 /* 1486 * This routine introduces a locking order for gswq->gs_lock followed 1487 * by cpu_lock. 1488 * This means that no consumer of the k-api should hold cpu_lock when calling 1489 * k-api routines. 1490 */ 1491 static void 1492 compute_min_max_threads() 1493 { 1494 psetid_t psid = PS_MYID; 1495 1496 mutex_enter(&gswq->gs_lock); 1497 if (cpupart_get_cpus(&psid, NULL, (uint_t *)&kcf_minthreads) != 0) { 1498 cmn_err(CE_WARN, "kcf:compute_min_max_threads cpupart_get_cpus:" 1499 " failed, setting kcf_minthreads to 1"); 1500 kcf_minthreads = 1; 1501 } 1502 kcf_maxthreads = kcf_thr_multiple * kcf_minthreads; 1503 gswq->gs_maxjobs = kcf_maxthreads * crypto_taskq_maxalloc; 1504 mutex_exit(&gswq->gs_lock); 1505 } 1506 1507 /* 1508 * This is the main routine of the failover kernel thread. 1509 * If there are any threads in the pool we sleep. The last thread in the 1510 * pool to exit will signal us to get to work. We get back to sleep 1511 * once we detect that the pool has threads. 1512 * 1513 * Note that in the hand-off from us to a pool thread we get to run once. 1514 * Since this hand-off is a rare event this should be fine. 1515 */ 1516 static void 1517 kcf_failover_thread() 1518 { 1519 int error = 0; 1520 kcf_context_t *ictx; 1521 kcf_areq_node_t *req; 1522 callb_cpr_t cpr_info; 1523 kmutex_t cpr_lock; 1524 static boolean_t is_logged = B_FALSE; 1525 1526 mutex_init(&cpr_lock, NULL, MUTEX_DEFAULT, NULL); 1527 CALLB_CPR_INIT(&cpr_info, &cpr_lock, callb_generic_cpr, 1528 "kcf_failover_thread"); 1529 1530 for (;;) { 1531 /* 1532 * Wait if there are any threads are in the pool. 1533 */ 1534 if (kcfpool->kp_threads > 0) { 1535 mutex_enter(&cpr_lock); 1536 CALLB_CPR_SAFE_BEGIN(&cpr_info); 1537 mutex_exit(&cpr_lock); 1538 1539 mutex_enter(&kcfpool->kp_thread_lock); 1540 cv_wait(&kcfpool->kp_nothr_cv, 1541 &kcfpool->kp_thread_lock); 1542 mutex_exit(&kcfpool->kp_thread_lock); 1543 1544 mutex_enter(&cpr_lock); 1545 CALLB_CPR_SAFE_END(&cpr_info, &cpr_lock); 1546 mutex_exit(&cpr_lock); 1547 is_logged = B_FALSE; 1548 } 1549 1550 /* 1551 * Get the requests from the queue and wait if needed. 1552 */ 1553 mutex_enter(&gswq->gs_lock); 1554 1555 while ((req = kcf_dequeue()) == NULL) { 1556 mutex_enter(&cpr_lock); 1557 CALLB_CPR_SAFE_BEGIN(&cpr_info); 1558 mutex_exit(&cpr_lock); 1559 1560 KCF_ATOMIC_INCR(kcfpool->kp_idlethreads); 1561 cv_wait(&gswq->gs_cv, &gswq->gs_lock); 1562 KCF_ATOMIC_DECR(kcfpool->kp_idlethreads); 1563 1564 mutex_enter(&cpr_lock); 1565 CALLB_CPR_SAFE_END(&cpr_info, &cpr_lock); 1566 mutex_exit(&cpr_lock); 1567 } 1568 1569 mutex_exit(&gswq->gs_lock); 1570 1571 /* 1572 * We check the kp_threads since kcfd could have started 1573 * while we are waiting on the global software queue. 1574 */ 1575 if (kcfpool->kp_threads <= 0 && !is_logged) { 1576 cmn_err(CE_WARN, "kcfd is not running. Please check " 1577 "and restart kcfd. Using the failover kernel " 1578 "thread for now.\n"); 1579 is_logged = B_TRUE; 1580 } 1581 1582 /* 1583 * Get to work on the request. 1584 */ 1585 ictx = req->an_context; 1586 mutex_enter(&req->an_lock); 1587 req->an_state = REQ_INPROGRESS; 1588 mutex_exit(&req->an_lock); 1589 1590 error = common_submit_request(req->an_provider, ictx ? 1591 &ictx->kc_glbl_ctx : NULL, &req->an_params, req); 1592 1593 kcf_aop_done(req, error); 1594 } 1595 } 1596 1597 /* 1598 * Insert the async request in the hash table after assigning it 1599 * an ID. Returns the ID. 1600 * 1601 * The ID is used by the caller to pass as an argument to a 1602 * cancel_req() routine later. 1603 */ 1604 static crypto_req_id_t 1605 kcf_reqid_insert(kcf_areq_node_t *areq) 1606 { 1607 int indx; 1608 crypto_req_id_t id; 1609 kcf_areq_node_t *headp; 1610 kcf_reqid_table_t *rt = 1611 kcf_reqid_table[CPU->cpu_seqid & REQID_TABLE_MASK]; 1612 1613 mutex_enter(&rt->rt_lock); 1614 1615 rt->rt_curid = id = 1616 (rt->rt_curid - REQID_COUNTER_LOW) | REQID_COUNTER_HIGH; 1617 SET_REQID(areq, id); 1618 indx = REQID_HASH(id); 1619 headp = areq->an_idnext = rt->rt_idhash[indx]; 1620 areq->an_idprev = NULL; 1621 if (headp != NULL) 1622 headp->an_idprev = areq; 1623 1624 rt->rt_idhash[indx] = areq; 1625 mutex_exit(&rt->rt_lock); 1626 1627 return (id); 1628 } 1629 1630 /* 1631 * Delete the async request from the hash table. 1632 */ 1633 static void 1634 kcf_reqid_delete(kcf_areq_node_t *areq) 1635 { 1636 int indx; 1637 kcf_areq_node_t *nextp, *prevp; 1638 crypto_req_id_t id = GET_REQID(areq); 1639 kcf_reqid_table_t *rt; 1640 1641 rt = kcf_reqid_table[id & REQID_TABLE_MASK]; 1642 indx = REQID_HASH(id); 1643 1644 mutex_enter(&rt->rt_lock); 1645 1646 nextp = areq->an_idnext; 1647 prevp = areq->an_idprev; 1648 if (nextp != NULL) 1649 nextp->an_idprev = prevp; 1650 if (prevp != NULL) 1651 prevp->an_idnext = nextp; 1652 else 1653 rt->rt_idhash[indx] = nextp; 1654 1655 SET_REQID(areq, 0); 1656 cv_broadcast(&areq->an_done); 1657 1658 mutex_exit(&rt->rt_lock); 1659 } 1660 1661 /* 1662 * Cancel a single asynchronous request. 1663 * 1664 * We guarantee that no problems will result from calling 1665 * crypto_cancel_req() for a request which is either running, or 1666 * has already completed. We remove the request from any queues 1667 * if it is possible. We wait for request completion if the 1668 * request is dispatched to a provider. 1669 * 1670 * Calling context: 1671 * Can be called from user context only. 1672 * 1673 * NOTE: We acquire the following locks in this routine (in order): 1674 * - rt_lock (kcf_reqid_table_t) 1675 * - gswq->gs_lock 1676 * - areq->an_lock 1677 * - ictx->kc_in_use_lock (from kcf_removereq_in_ctxchain()) 1678 * 1679 * This locking order MUST be maintained in code every where else. 1680 */ 1681 void 1682 crypto_cancel_req(crypto_req_id_t id) 1683 { 1684 int indx; 1685 kcf_areq_node_t *areq; 1686 kcf_provider_desc_t *pd; 1687 kcf_context_t *ictx; 1688 kcf_reqid_table_t *rt; 1689 1690 rt = kcf_reqid_table[id & REQID_TABLE_MASK]; 1691 indx = REQID_HASH(id); 1692 1693 mutex_enter(&rt->rt_lock); 1694 for (areq = rt->rt_idhash[indx]; areq; areq = areq->an_idnext) { 1695 if (GET_REQID(areq) == id) { 1696 /* 1697 * We found the request. It is either still waiting 1698 * in the framework queues or running at the provider. 1699 */ 1700 pd = areq->an_provider; 1701 ASSERT(pd != NULL); 1702 1703 switch (pd->pd_prov_type) { 1704 case CRYPTO_SW_PROVIDER: 1705 mutex_enter(&gswq->gs_lock); 1706 mutex_enter(&areq->an_lock); 1707 1708 /* This request can be safely canceled. */ 1709 if (areq->an_state <= REQ_WAITING) { 1710 /* Remove from gswq, global software queue. */ 1711 kcf_remove_node(areq); 1712 if ((ictx = areq->an_context) != NULL) 1713 kcf_removereq_in_ctxchain(ictx, areq); 1714 1715 mutex_exit(&areq->an_lock); 1716 mutex_exit(&gswq->gs_lock); 1717 mutex_exit(&rt->rt_lock); 1718 1719 /* Remove areq from hash table and free it. */ 1720 kcf_reqid_delete(areq); 1721 KCF_AREQ_REFRELE(areq); 1722 return; 1723 } 1724 1725 mutex_exit(&areq->an_lock); 1726 mutex_exit(&gswq->gs_lock); 1727 break; 1728 1729 case CRYPTO_HW_PROVIDER: 1730 /* 1731 * There is no interface to remove an entry 1732 * once it is on the taskq. So, we do not do 1733 * any thing for a hardware provider. 1734 */ 1735 break; 1736 } 1737 1738 /* 1739 * The request is running. Wait for the request completion 1740 * to notify us. 1741 */ 1742 KCF_AREQ_REFHOLD(areq); 1743 while (GET_REQID(areq) == id) 1744 cv_wait(&areq->an_done, &rt->rt_lock); 1745 KCF_AREQ_REFRELE(areq); 1746 break; 1747 } 1748 } 1749 1750 mutex_exit(&rt->rt_lock); 1751 } 1752 1753 /* 1754 * Cancel all asynchronous requests associated with the 1755 * passed in crypto context and free it. 1756 * 1757 * A client SHOULD NOT call this routine after calling a crypto_*_final 1758 * routine. This routine is called only during intermediate operations. 1759 * The client should not use the crypto context after this function returns 1760 * since we destroy it. 1761 * 1762 * Calling context: 1763 * Can be called from user context only. 1764 */ 1765 void 1766 crypto_cancel_ctx(crypto_context_t ctx) 1767 { 1768 kcf_context_t *ictx; 1769 kcf_areq_node_t *areq; 1770 1771 if (ctx == NULL) 1772 return; 1773 1774 ictx = (kcf_context_t *)((crypto_ctx_t *)ctx)->cc_framework_private; 1775 1776 mutex_enter(&ictx->kc_in_use_lock); 1777 1778 /* Walk the chain and cancel each request */ 1779 while ((areq = ictx->kc_req_chain_first) != NULL) { 1780 /* 1781 * We have to drop the lock here as we may have 1782 * to wait for request completion. We hold the 1783 * request before dropping the lock though, so that it 1784 * won't be freed underneath us. 1785 */ 1786 KCF_AREQ_REFHOLD(areq); 1787 mutex_exit(&ictx->kc_in_use_lock); 1788 1789 crypto_cancel_req(GET_REQID(areq)); 1790 KCF_AREQ_REFRELE(areq); 1791 1792 mutex_enter(&ictx->kc_in_use_lock); 1793 } 1794 1795 mutex_exit(&ictx->kc_in_use_lock); 1796 KCF_CONTEXT_REFRELE(ictx); 1797 } 1798 1799 /* 1800 * Update kstats. 1801 */ 1802 static int 1803 kcf_misc_kstat_update(kstat_t *ksp, int rw) 1804 { 1805 uint_t tcnt; 1806 kcf_stats_t *ks_data; 1807 1808 if (rw == KSTAT_WRITE) 1809 return (EACCES); 1810 1811 ks_data = ksp->ks_data; 1812 1813 ks_data->ks_thrs_in_pool.value.ui32 = kcfpool->kp_threads; 1814 /* 1815 * The failover thread is counted in kp_idlethreads in 1816 * some corner cases. This is done to avoid doing more checks 1817 * when submitting a request. We account for those cases below. 1818 */ 1819 if ((tcnt = kcfpool->kp_idlethreads) == (kcfpool->kp_threads + 1)) 1820 tcnt--; 1821 ks_data->ks_idle_thrs.value.ui32 = tcnt; 1822 ks_data->ks_minthrs.value.ui32 = kcf_minthreads; 1823 ks_data->ks_maxthrs.value.ui32 = kcf_maxthreads; 1824 ks_data->ks_swq_njobs.value.ui32 = gswq->gs_njobs; 1825 ks_data->ks_swq_maxjobs.value.ui32 = gswq->gs_maxjobs; 1826 ks_data->ks_taskq_minalloc.value.ui32 = crypto_taskq_minalloc; 1827 ks_data->ks_taskq_maxalloc.value.ui32 = crypto_taskq_maxalloc; 1828 1829 return (0); 1830 } 1831 1832 /* 1833 * Allocate and initiatize a kcf_dual_req, used for saving the arguments of 1834 * a dual operation or an atomic operation that has to be internally 1835 * simulated with multiple single steps. 1836 * crq determines the memory allocation flags. 1837 */ 1838 1839 kcf_dual_req_t * 1840 kcf_alloc_req(crypto_call_req_t *crq) 1841 { 1842 kcf_dual_req_t *kcr; 1843 1844 kcr = kmem_alloc(sizeof (kcf_dual_req_t), KCF_KMFLAG(crq)); 1845 1846 if (kcr == NULL) 1847 return (NULL); 1848 1849 /* Copy the whole crypto_call_req struct, as it isn't persistant */ 1850 if (crq != NULL) 1851 kcr->kr_callreq = *crq; 1852 else 1853 bzero(&(kcr->kr_callreq), sizeof (crypto_call_req_t)); 1854 kcr->kr_areq = NULL; 1855 kcr->kr_saveoffset = 0; 1856 kcr->kr_savelen = 0; 1857 1858 return (kcr); 1859 } 1860 1861 /* 1862 * Callback routine for the next part of a simulated dual part. 1863 * Schedules the next step. 1864 * 1865 * This routine can be called from interrupt context. 1866 */ 1867 void 1868 kcf_next_req(void *next_req_arg, int status) 1869 { 1870 kcf_dual_req_t *next_req = (kcf_dual_req_t *)next_req_arg; 1871 kcf_req_params_t *params = &(next_req->kr_params); 1872 kcf_areq_node_t *areq = next_req->kr_areq; 1873 int error = status; 1874 kcf_provider_desc_t *pd; 1875 crypto_dual_data_t *ct; 1876 1877 /* Stop the processing if an error occured at this step */ 1878 if (error != CRYPTO_SUCCESS) { 1879 out: 1880 areq->an_reqarg = next_req->kr_callreq; 1881 KCF_AREQ_REFRELE(areq); 1882 kmem_free(next_req, sizeof (kcf_dual_req_t)); 1883 areq->an_isdual = B_FALSE; 1884 kcf_aop_done(areq, error); 1885 return; 1886 } 1887 1888 switch (params->rp_opgrp) { 1889 case KCF_OG_MAC: { 1890 1891 /* 1892 * The next req is submitted with the same reqid as the 1893 * first part. The consumer only got back that reqid, and 1894 * should still be able to cancel the operation during its 1895 * second step. 1896 */ 1897 kcf_mac_ops_params_t *mops = &(params->rp_u.mac_params); 1898 crypto_ctx_template_t mac_tmpl; 1899 kcf_mech_entry_t *me; 1900 1901 ct = (crypto_dual_data_t *)mops->mo_data; 1902 mac_tmpl = (crypto_ctx_template_t)mops->mo_templ; 1903 1904 /* No expected recoverable failures, so no retry list */ 1905 pd = kcf_get_mech_provider(mops->mo_framework_mechtype, 1906 &me, &error, NULL, CRYPTO_FG_MAC_ATOMIC, 1907 (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), ct->dd_len2); 1908 1909 if (pd == NULL) { 1910 error = CRYPTO_MECH_NOT_SUPPORTED; 1911 goto out; 1912 } 1913 /* Validate the MAC context template here */ 1914 if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) && 1915 (mac_tmpl != NULL)) { 1916 kcf_ctx_template_t *ctx_mac_tmpl; 1917 1918 ctx_mac_tmpl = (kcf_ctx_template_t *)mac_tmpl; 1919 1920 if (ctx_mac_tmpl->ct_generation != me->me_gen_swprov) { 1921 KCF_PROV_REFRELE(pd); 1922 error = CRYPTO_OLD_CTX_TEMPLATE; 1923 goto out; 1924 } 1925 mops->mo_templ = ctx_mac_tmpl->ct_prov_tmpl; 1926 } 1927 1928 break; 1929 } 1930 case KCF_OG_DECRYPT: { 1931 kcf_decrypt_ops_params_t *dcrops = 1932 &(params->rp_u.decrypt_params); 1933 1934 ct = (crypto_dual_data_t *)dcrops->dop_ciphertext; 1935 /* No expected recoverable failures, so no retry list */ 1936 pd = kcf_get_mech_provider(dcrops->dop_framework_mechtype, 1937 NULL, &error, NULL, CRYPTO_FG_DECRYPT_ATOMIC, 1938 (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), ct->dd_len1); 1939 1940 if (pd == NULL) { 1941 error = CRYPTO_MECH_NOT_SUPPORTED; 1942 goto out; 1943 } 1944 break; 1945 } 1946 } 1947 1948 /* The second step uses len2 and offset2 of the dual_data */ 1949 next_req->kr_saveoffset = ct->dd_offset1; 1950 next_req->kr_savelen = ct->dd_len1; 1951 ct->dd_offset1 = ct->dd_offset2; 1952 ct->dd_len1 = ct->dd_len2; 1953 1954 /* preserve if the caller is restricted */ 1955 if (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED) { 1956 areq->an_reqarg.cr_flag = CRYPTO_RESTRICTED; 1957 } else { 1958 areq->an_reqarg.cr_flag = 0; 1959 } 1960 1961 areq->an_reqarg.cr_callback_func = kcf_last_req; 1962 areq->an_reqarg.cr_callback_arg = next_req; 1963 areq->an_isdual = B_TRUE; 1964 1965 /* 1966 * We would like to call kcf_submit_request() here. But, 1967 * that is not possible as that routine allocates a new 1968 * kcf_areq_node_t request structure, while we need to 1969 * reuse the existing request structure. 1970 */ 1971 switch (pd->pd_prov_type) { 1972 case CRYPTO_SW_PROVIDER: 1973 error = common_submit_request(pd, NULL, params, 1974 KCF_RHNDL(KM_NOSLEEP)); 1975 break; 1976 1977 case CRYPTO_HW_PROVIDER: { 1978 kcf_provider_desc_t *old_pd; 1979 taskq_t *taskq = pd->pd_sched_info.ks_taskq; 1980 1981 /* 1982 * Set the params for the second step in the 1983 * dual-ops. 1984 */ 1985 areq->an_params = *params; 1986 old_pd = areq->an_provider; 1987 KCF_PROV_REFRELE(old_pd); 1988 KCF_PROV_REFHOLD(pd); 1989 areq->an_provider = pd; 1990 1991 /* 1992 * Note that we have to do a taskq_dispatch() 1993 * here as we may be in interrupt context. 1994 */ 1995 if (taskq_dispatch(taskq, process_req_hwp, areq, 1996 TQ_NOSLEEP) == (taskqid_t)0) { 1997 error = CRYPTO_HOST_MEMORY; 1998 } else { 1999 error = CRYPTO_QUEUED; 2000 } 2001 break; 2002 } 2003 } 2004 2005 /* 2006 * We have to release the holds on the request and the provider 2007 * in all cases. 2008 */ 2009 KCF_AREQ_REFRELE(areq); 2010 KCF_PROV_REFRELE(pd); 2011 2012 if (error != CRYPTO_QUEUED) { 2013 /* restore, clean up, and invoke the client's callback */ 2014 2015 ct->dd_offset1 = next_req->kr_saveoffset; 2016 ct->dd_len1 = next_req->kr_savelen; 2017 areq->an_reqarg = next_req->kr_callreq; 2018 kmem_free(next_req, sizeof (kcf_dual_req_t)); 2019 areq->an_isdual = B_FALSE; 2020 kcf_aop_done(areq, error); 2021 } 2022 } 2023 2024 /* 2025 * Last part of an emulated dual operation. 2026 * Clean up and restore ... 2027 */ 2028 void 2029 kcf_last_req(void *last_req_arg, int status) 2030 { 2031 kcf_dual_req_t *last_req = (kcf_dual_req_t *)last_req_arg; 2032 2033 kcf_req_params_t *params = &(last_req->kr_params); 2034 kcf_areq_node_t *areq = last_req->kr_areq; 2035 crypto_dual_data_t *ct; 2036 2037 switch (params->rp_opgrp) { 2038 case KCF_OG_MAC: { 2039 kcf_mac_ops_params_t *mops = &(params->rp_u.mac_params); 2040 2041 ct = (crypto_dual_data_t *)mops->mo_data; 2042 break; 2043 } 2044 case KCF_OG_DECRYPT: { 2045 kcf_decrypt_ops_params_t *dcrops = 2046 &(params->rp_u.decrypt_params); 2047 2048 ct = (crypto_dual_data_t *)dcrops->dop_ciphertext; 2049 break; 2050 } 2051 } 2052 ct->dd_offset1 = last_req->kr_saveoffset; 2053 ct->dd_len1 = last_req->kr_savelen; 2054 2055 /* The submitter used kcf_last_req as its callback */ 2056 2057 if (areq == NULL) { 2058 crypto_call_req_t *cr = &last_req->kr_callreq; 2059 2060 (*(cr->cr_callback_func))(cr->cr_callback_arg, status); 2061 kmem_free(last_req, sizeof (kcf_dual_req_t)); 2062 return; 2063 } 2064 areq->an_reqarg = last_req->kr_callreq; 2065 KCF_AREQ_REFRELE(areq); 2066 kmem_free(last_req, sizeof (kcf_dual_req_t)); 2067 areq->an_isdual = B_FALSE; 2068 kcf_aop_done(areq, status); 2069 } 2070