1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * Copyright 2010 Nexenta Systems, Inc. All rights reserved. 27 */ 28 29 /* 30 * This file contains the core framework routines for the 31 * kernel cryptographic framework. These routines are at the 32 * layer, between the kernel API/ioctls and the SPI. 33 */ 34 35 #include <sys/types.h> 36 #include <sys/errno.h> 37 #include <sys/kmem.h> 38 #include <sys/proc.h> 39 #include <sys/cpuvar.h> 40 #include <sys/cpupart.h> 41 #include <sys/ksynch.h> 42 #include <sys/callb.h> 43 #include <sys/cmn_err.h> 44 #include <sys/systm.h> 45 #include <sys/sysmacros.h> 46 #include <sys/kstat.h> 47 #include <sys/crypto/common.h> 48 #include <sys/crypto/impl.h> 49 #include <sys/crypto/sched_impl.h> 50 #include <sys/crypto/api.h> 51 #include <sys/crypto/spi.h> 52 #include <sys/taskq_impl.h> 53 #include <sys/ddi.h> 54 #include <sys/sunddi.h> 55 56 57 kcf_global_swq_t *gswq; /* Global software queue */ 58 59 /* Thread pool related variables */ 60 static kcf_pool_t *kcfpool; /* Thread pool of kcfd LWPs */ 61 int kcf_maxthreads = 2; 62 int kcf_minthreads = 1; 63 int kcf_thr_multiple = 2; /* Boot-time tunable for experimentation */ 64 static ulong_t kcf_idlethr_timeout; 65 static boolean_t kcf_sched_running = B_FALSE; 66 #define KCF_DEFAULT_THRTIMEOUT 60000000 /* 60 seconds */ 67 68 /* kmem caches used by the scheduler */ 69 static struct kmem_cache *kcf_sreq_cache; 70 static struct kmem_cache *kcf_areq_cache; 71 static struct kmem_cache *kcf_context_cache; 72 73 /* Global request ID table */ 74 static kcf_reqid_table_t *kcf_reqid_table[REQID_TABLES]; 75 76 /* KCF stats. Not protected. */ 77 static kcf_stats_t kcf_ksdata = { 78 { "total threads in pool", KSTAT_DATA_UINT32}, 79 { "idle threads in pool", KSTAT_DATA_UINT32}, 80 { "min threads in pool", KSTAT_DATA_UINT32}, 81 { "max threads in pool", KSTAT_DATA_UINT32}, 82 { "requests in gswq", KSTAT_DATA_UINT32}, 83 { "max requests in gswq", KSTAT_DATA_UINT32}, 84 { "threads for HW taskq", KSTAT_DATA_UINT32}, 85 { "minalloc for HW taskq", KSTAT_DATA_UINT32}, 86 { "maxalloc for HW taskq", KSTAT_DATA_UINT32} 87 }; 88 89 static kstat_t *kcf_misc_kstat = NULL; 90 ulong_t kcf_swprov_hndl = 0; 91 92 static kcf_areq_node_t *kcf_areqnode_alloc(kcf_provider_desc_t *, 93 kcf_context_t *, crypto_call_req_t *, kcf_req_params_t *, boolean_t); 94 static int kcf_disp_sw_request(kcf_areq_node_t *); 95 static void process_req_hwp(void *); 96 static kcf_areq_node_t *kcf_dequeue(void); 97 static int kcf_enqueue(kcf_areq_node_t *); 98 static void kcfpool_alloc(void); 99 static void kcf_reqid_delete(kcf_areq_node_t *areq); 100 static crypto_req_id_t kcf_reqid_insert(kcf_areq_node_t *areq); 101 static int kcf_misc_kstat_update(kstat_t *ksp, int rw); 102 static void compute_min_max_threads(void); 103 static void kcfpool_svc(void *); 104 static void kcfpoold(void *); 105 106 107 /* 108 * Create a new context. 109 */ 110 crypto_ctx_t * 111 kcf_new_ctx(crypto_call_req_t *crq, kcf_provider_desc_t *pd, 112 crypto_session_id_t sid) 113 { 114 crypto_ctx_t *ctx; 115 kcf_context_t *kcf_ctx; 116 117 kcf_ctx = kmem_cache_alloc(kcf_context_cache, 118 (crq == NULL) ? KM_SLEEP : KM_NOSLEEP); 119 if (kcf_ctx == NULL) 120 return (NULL); 121 122 /* initialize the context for the consumer */ 123 kcf_ctx->kc_refcnt = 1; 124 kcf_ctx->kc_req_chain_first = NULL; 125 kcf_ctx->kc_req_chain_last = NULL; 126 kcf_ctx->kc_secondctx = NULL; 127 KCF_PROV_REFHOLD(pd); 128 kcf_ctx->kc_prov_desc = pd; 129 kcf_ctx->kc_sw_prov_desc = NULL; 130 kcf_ctx->kc_mech = NULL; 131 132 ctx = &kcf_ctx->kc_glbl_ctx; 133 ctx->cc_provider = pd->pd_prov_handle; 134 ctx->cc_session = sid; 135 ctx->cc_provider_private = NULL; 136 ctx->cc_framework_private = (void *)kcf_ctx; 137 ctx->cc_flags = 0; 138 ctx->cc_opstate = NULL; 139 140 return (ctx); 141 } 142 143 /* 144 * Allocate a new async request node. 145 * 146 * ictx - Framework private context pointer 147 * crq - Has callback function and argument. Should be non NULL. 148 * req - The parameters to pass to the SPI 149 */ 150 static kcf_areq_node_t * 151 kcf_areqnode_alloc(kcf_provider_desc_t *pd, kcf_context_t *ictx, 152 crypto_call_req_t *crq, kcf_req_params_t *req, boolean_t isdual) 153 { 154 kcf_areq_node_t *arptr, *areq; 155 156 ASSERT(crq != NULL); 157 arptr = kmem_cache_alloc(kcf_areq_cache, KM_NOSLEEP); 158 if (arptr == NULL) 159 return (NULL); 160 161 arptr->an_state = REQ_ALLOCATED; 162 arptr->an_reqarg = *crq; 163 arptr->an_params = *req; 164 arptr->an_context = ictx; 165 arptr->an_isdual = isdual; 166 167 arptr->an_next = arptr->an_prev = NULL; 168 KCF_PROV_REFHOLD(pd); 169 arptr->an_provider = pd; 170 arptr->an_tried_plist = NULL; 171 arptr->an_refcnt = 1; 172 arptr->an_idnext = arptr->an_idprev = NULL; 173 174 /* 175 * Requests for context-less operations do not use the 176 * fields - an_is_my_turn, and an_ctxchain_next. 177 */ 178 if (ictx == NULL) 179 return (arptr); 180 181 KCF_CONTEXT_REFHOLD(ictx); 182 /* 183 * Chain this request to the context. 184 */ 185 mutex_enter(&ictx->kc_in_use_lock); 186 arptr->an_ctxchain_next = NULL; 187 if ((areq = ictx->kc_req_chain_last) == NULL) { 188 arptr->an_is_my_turn = B_TRUE; 189 ictx->kc_req_chain_last = 190 ictx->kc_req_chain_first = arptr; 191 } else { 192 ASSERT(ictx->kc_req_chain_first != NULL); 193 arptr->an_is_my_turn = B_FALSE; 194 /* Insert the new request to the end of the chain. */ 195 areq->an_ctxchain_next = arptr; 196 ictx->kc_req_chain_last = arptr; 197 } 198 mutex_exit(&ictx->kc_in_use_lock); 199 200 return (arptr); 201 } 202 203 /* 204 * Queue the request node and do one of the following: 205 * - If there is an idle thread signal it to run. 206 * - Else, signal the creator thread to possibly create more threads. 207 */ 208 static int 209 kcf_disp_sw_request(kcf_areq_node_t *areq) 210 { 211 int err; 212 213 if ((err = kcf_enqueue(areq)) != 0) 214 return (err); 215 216 if (kcfpool->kp_idlethreads > 0) { 217 /* Signal an idle thread to run */ 218 mutex_enter(&gswq->gs_lock); 219 cv_signal(&gswq->gs_cv); 220 mutex_exit(&gswq->gs_lock); 221 222 return (CRYPTO_QUEUED); 223 } 224 225 /* Signal the creator thread for more threads */ 226 mutex_enter(&kcfpool->kp_lock); 227 cv_signal(&kcfpool->kp_cv); 228 mutex_exit(&kcfpool->kp_lock); 229 230 return (CRYPTO_QUEUED); 231 } 232 233 /* 234 * This routine is called by the taskq associated with 235 * each hardware provider. We notify the kernel consumer 236 * via the callback routine in case of CRYPTO_SUCCESS or 237 * a failure. 238 * 239 * A request can be of type kcf_areq_node_t or of type 240 * kcf_sreq_node_t. 241 */ 242 static void 243 process_req_hwp(void *ireq) 244 { 245 int error = 0; 246 crypto_ctx_t *ctx; 247 kcf_call_type_t ctype; 248 kcf_provider_desc_t *pd; 249 kcf_areq_node_t *areq = (kcf_areq_node_t *)ireq; 250 kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)ireq; 251 kcf_prov_cpu_t *mp; 252 253 pd = ((ctype = GET_REQ_TYPE(ireq)) == CRYPTO_SYNCH) ? 254 sreq->sn_provider : areq->an_provider; 255 256 /* 257 * Wait if flow control is in effect for the provider. A 258 * CRYPTO_PROVIDER_READY or CRYPTO_PROVIDER_FAILED 259 * notification will signal us. We also get signaled if 260 * the provider is unregistering. 261 */ 262 if (pd->pd_state == KCF_PROV_BUSY) { 263 mutex_enter(&pd->pd_lock); 264 while (pd->pd_state == KCF_PROV_BUSY) 265 cv_wait(&pd->pd_resume_cv, &pd->pd_lock); 266 mutex_exit(&pd->pd_lock); 267 } 268 269 /* 270 * Bump the internal reference count while the request is being 271 * processed. This is how we know when it's safe to unregister 272 * a provider. This step must precede the pd_state check below. 273 */ 274 mp = &(pd->pd_percpu_bins[CPU_SEQID]); 275 KCF_PROV_JOB_HOLD(mp); 276 277 /* 278 * Fail the request if the provider has failed. We return a 279 * recoverable error and the notified clients attempt any 280 * recovery. For async clients this is done in kcf_aop_done() 281 * and for sync clients it is done in the k-api routines. 282 */ 283 if (pd->pd_state >= KCF_PROV_FAILED) { 284 error = CRYPTO_DEVICE_ERROR; 285 goto bail; 286 } 287 288 if (ctype == CRYPTO_SYNCH) { 289 mutex_enter(&sreq->sn_lock); 290 sreq->sn_state = REQ_INPROGRESS; 291 sreq->sn_mp = mp; 292 mutex_exit(&sreq->sn_lock); 293 294 ctx = sreq->sn_context ? &sreq->sn_context->kc_glbl_ctx : NULL; 295 error = common_submit_request(sreq->sn_provider, ctx, 296 sreq->sn_params, sreq); 297 } else { 298 kcf_context_t *ictx; 299 ASSERT(ctype == CRYPTO_ASYNCH); 300 301 /* 302 * We are in the per-hardware provider thread context and 303 * hence can sleep. Note that the caller would have done 304 * a taskq_dispatch(..., TQ_NOSLEEP) and would have returned. 305 */ 306 ctx = (ictx = areq->an_context) ? &ictx->kc_glbl_ctx : NULL; 307 308 mutex_enter(&areq->an_lock); 309 /* 310 * We need to maintain ordering for multi-part requests. 311 * an_is_my_turn is set to B_TRUE initially for a request 312 * when it is enqueued and there are no other requests 313 * for that context. It is set later from kcf_aop_done() when 314 * the request before us in the chain of requests for the 315 * context completes. We get signaled at that point. 316 */ 317 if (ictx != NULL) { 318 ASSERT(ictx->kc_prov_desc == areq->an_provider); 319 320 while (areq->an_is_my_turn == B_FALSE) { 321 cv_wait(&areq->an_turn_cv, &areq->an_lock); 322 } 323 } 324 areq->an_state = REQ_INPROGRESS; 325 areq->an_mp = mp; 326 mutex_exit(&areq->an_lock); 327 328 error = common_submit_request(areq->an_provider, ctx, 329 &areq->an_params, areq); 330 } 331 332 bail: 333 if (error == CRYPTO_QUEUED) { 334 /* 335 * The request is queued by the provider and we should 336 * get a crypto_op_notification() from the provider later. 337 * We notify the consumer at that time. 338 */ 339 return; 340 } else { /* CRYPTO_SUCCESS or other failure */ 341 KCF_PROV_JOB_RELE(mp); 342 if (ctype == CRYPTO_SYNCH) 343 kcf_sop_done(sreq, error); 344 else 345 kcf_aop_done(areq, error); 346 } 347 } 348 349 /* 350 * This routine checks if a request can be retried on another 351 * provider. If true, mech1 is initialized to point to the mechanism 352 * structure. mech2 is also initialized in case of a dual operation. fg 353 * is initialized to the correct crypto_func_group_t bit flag. They are 354 * initialized by this routine, so that the caller can pass them to a 355 * kcf_get_mech_provider() or kcf_get_dual_provider() with no further change. 356 * 357 * We check that the request is for a init or atomic routine and that 358 * it is for one of the operation groups used from k-api . 359 */ 360 static boolean_t 361 can_resubmit(kcf_areq_node_t *areq, crypto_mechanism_t **mech1, 362 crypto_mechanism_t **mech2, crypto_func_group_t *fg) 363 { 364 kcf_req_params_t *params; 365 kcf_op_type_t optype; 366 367 params = &areq->an_params; 368 optype = params->rp_optype; 369 370 if (!(IS_INIT_OP(optype) || IS_ATOMIC_OP(optype))) 371 return (B_FALSE); 372 373 switch (params->rp_opgrp) { 374 case KCF_OG_DIGEST: { 375 kcf_digest_ops_params_t *dops = ¶ms->rp_u.digest_params; 376 377 dops->do_mech.cm_type = dops->do_framework_mechtype; 378 *mech1 = &dops->do_mech; 379 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_DIGEST : 380 CRYPTO_FG_DIGEST_ATOMIC; 381 break; 382 } 383 384 case KCF_OG_MAC: { 385 kcf_mac_ops_params_t *mops = ¶ms->rp_u.mac_params; 386 387 mops->mo_mech.cm_type = mops->mo_framework_mechtype; 388 *mech1 = &mops->mo_mech; 389 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_MAC : 390 CRYPTO_FG_MAC_ATOMIC; 391 break; 392 } 393 394 case KCF_OG_SIGN: { 395 kcf_sign_ops_params_t *sops = ¶ms->rp_u.sign_params; 396 397 sops->so_mech.cm_type = sops->so_framework_mechtype; 398 *mech1 = &sops->so_mech; 399 switch (optype) { 400 case KCF_OP_INIT: 401 *fg = CRYPTO_FG_SIGN; 402 break; 403 case KCF_OP_ATOMIC: 404 *fg = CRYPTO_FG_SIGN_ATOMIC; 405 break; 406 default: 407 ASSERT(optype == KCF_OP_SIGN_RECOVER_ATOMIC); 408 *fg = CRYPTO_FG_SIGN_RECOVER_ATOMIC; 409 } 410 break; 411 } 412 413 case KCF_OG_VERIFY: { 414 kcf_verify_ops_params_t *vops = ¶ms->rp_u.verify_params; 415 416 vops->vo_mech.cm_type = vops->vo_framework_mechtype; 417 *mech1 = &vops->vo_mech; 418 switch (optype) { 419 case KCF_OP_INIT: 420 *fg = CRYPTO_FG_VERIFY; 421 break; 422 case KCF_OP_ATOMIC: 423 *fg = CRYPTO_FG_VERIFY_ATOMIC; 424 break; 425 default: 426 ASSERT(optype == KCF_OP_VERIFY_RECOVER_ATOMIC); 427 *fg = CRYPTO_FG_VERIFY_RECOVER_ATOMIC; 428 } 429 break; 430 } 431 432 case KCF_OG_ENCRYPT: { 433 kcf_encrypt_ops_params_t *eops = ¶ms->rp_u.encrypt_params; 434 435 eops->eo_mech.cm_type = eops->eo_framework_mechtype; 436 *mech1 = &eops->eo_mech; 437 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_ENCRYPT : 438 CRYPTO_FG_ENCRYPT_ATOMIC; 439 break; 440 } 441 442 case KCF_OG_DECRYPT: { 443 kcf_decrypt_ops_params_t *dcrops = ¶ms->rp_u.decrypt_params; 444 445 dcrops->dop_mech.cm_type = dcrops->dop_framework_mechtype; 446 *mech1 = &dcrops->dop_mech; 447 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_DECRYPT : 448 CRYPTO_FG_DECRYPT_ATOMIC; 449 break; 450 } 451 452 case KCF_OG_ENCRYPT_MAC: { 453 kcf_encrypt_mac_ops_params_t *eops = 454 ¶ms->rp_u.encrypt_mac_params; 455 456 eops->em_encr_mech.cm_type = eops->em_framework_encr_mechtype; 457 *mech1 = &eops->em_encr_mech; 458 eops->em_mac_mech.cm_type = eops->em_framework_mac_mechtype; 459 *mech2 = &eops->em_mac_mech; 460 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_ENCRYPT_MAC : 461 CRYPTO_FG_ENCRYPT_MAC_ATOMIC; 462 break; 463 } 464 465 case KCF_OG_MAC_DECRYPT: { 466 kcf_mac_decrypt_ops_params_t *dops = 467 ¶ms->rp_u.mac_decrypt_params; 468 469 dops->md_mac_mech.cm_type = dops->md_framework_mac_mechtype; 470 *mech1 = &dops->md_mac_mech; 471 dops->md_decr_mech.cm_type = dops->md_framework_decr_mechtype; 472 *mech2 = &dops->md_decr_mech; 473 *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_MAC_DECRYPT : 474 CRYPTO_FG_MAC_DECRYPT_ATOMIC; 475 break; 476 } 477 478 default: 479 return (B_FALSE); 480 } 481 482 return (B_TRUE); 483 } 484 485 /* 486 * This routine is called when a request to a provider has failed 487 * with a recoverable error. This routine tries to find another provider 488 * and dispatches the request to the new provider, if one is available. 489 * We reuse the request structure. 490 * 491 * A return value of NULL from kcf_get_mech_provider() indicates 492 * we have tried the last provider. 493 */ 494 static int 495 kcf_resubmit_request(kcf_areq_node_t *areq) 496 { 497 int error = CRYPTO_FAILED; 498 kcf_context_t *ictx; 499 kcf_provider_desc_t *old_pd; 500 kcf_provider_desc_t *new_pd; 501 crypto_mechanism_t *mech1 = NULL, *mech2 = NULL; 502 crypto_mech_type_t prov_mt1, prov_mt2; 503 crypto_func_group_t fg; 504 505 if (!can_resubmit(areq, &mech1, &mech2, &fg)) 506 return (error); 507 508 old_pd = areq->an_provider; 509 /* 510 * Add old_pd to the list of providers already tried. 511 * We release the new hold on old_pd in kcf_free_triedlist(). 512 */ 513 if (kcf_insert_triedlist(&areq->an_tried_plist, old_pd, 514 KM_NOSLEEP | KCF_HOLD_PROV) == NULL) 515 return (error); 516 517 if (mech1 && !mech2) { 518 new_pd = kcf_get_mech_provider(mech1->cm_type, NULL, NULL, 519 &error, areq->an_tried_plist, fg, 0); 520 } else { 521 ASSERT(mech1 != NULL && mech2 != NULL); 522 523 new_pd = kcf_get_dual_provider(mech1, NULL, mech2, NULL, 524 NULL, &prov_mt1, 525 &prov_mt2, &error, areq->an_tried_plist, fg, fg, 0); 526 } 527 528 if (new_pd == NULL) 529 return (error); 530 531 /* 532 * We reuse the old context by resetting provider specific 533 * fields in it. 534 */ 535 if ((ictx = areq->an_context) != NULL) { 536 crypto_ctx_t *ctx; 537 538 ASSERT(old_pd == ictx->kc_prov_desc); 539 KCF_PROV_REFRELE(ictx->kc_prov_desc); 540 KCF_PROV_REFHOLD(new_pd); 541 ictx->kc_prov_desc = new_pd; 542 543 ctx = &ictx->kc_glbl_ctx; 544 ctx->cc_provider = new_pd->pd_prov_handle; 545 ctx->cc_session = new_pd->pd_sid; 546 ctx->cc_provider_private = NULL; 547 } 548 549 /* We reuse areq. by resetting the provider and context fields. */ 550 KCF_PROV_REFRELE(old_pd); 551 KCF_PROV_REFHOLD(new_pd); 552 areq->an_provider = new_pd; 553 mutex_enter(&areq->an_lock); 554 areq->an_state = REQ_WAITING; 555 mutex_exit(&areq->an_lock); 556 557 switch (new_pd->pd_prov_type) { 558 case CRYPTO_SW_PROVIDER: 559 error = kcf_disp_sw_request(areq); 560 break; 561 562 case CRYPTO_HW_PROVIDER: { 563 taskq_t *taskq = new_pd->pd_taskq; 564 565 if (taskq_dispatch(taskq, process_req_hwp, areq, TQ_NOSLEEP) == 566 (taskqid_t)0) { 567 error = CRYPTO_HOST_MEMORY; 568 } else { 569 error = CRYPTO_QUEUED; 570 } 571 572 break; 573 } 574 } 575 576 KCF_PROV_REFRELE(new_pd); 577 return (error); 578 } 579 580 #define EMPTY_TASKQ(tq) ((tq)->tq_task.tqent_next == &(tq)->tq_task) 581 582 /* 583 * Routine called by both ioctl and k-api. The consumer should 584 * bundle the parameters into a kcf_req_params_t structure. A bunch 585 * of macros are available in ops_impl.h for this bundling. They are: 586 * 587 * KCF_WRAP_DIGEST_OPS_PARAMS() 588 * KCF_WRAP_MAC_OPS_PARAMS() 589 * KCF_WRAP_ENCRYPT_OPS_PARAMS() 590 * KCF_WRAP_DECRYPT_OPS_PARAMS() ... etc. 591 * 592 * It is the caller's responsibility to free the ctx argument when 593 * appropriate. See the KCF_CONTEXT_COND_RELEASE macro for details. 594 */ 595 int 596 kcf_submit_request(kcf_provider_desc_t *pd, crypto_ctx_t *ctx, 597 crypto_call_req_t *crq, kcf_req_params_t *params, boolean_t cont) 598 { 599 int error; 600 kcf_areq_node_t *areq; 601 kcf_sreq_node_t *sreq; 602 kcf_context_t *kcf_ctx; 603 taskq_t *taskq; 604 kcf_prov_cpu_t *mp; 605 606 kcf_ctx = ctx ? (kcf_context_t *)ctx->cc_framework_private : NULL; 607 608 /* Synchronous cases */ 609 if (crq == NULL) { 610 switch (pd->pd_prov_type) { 611 case CRYPTO_SW_PROVIDER: 612 error = common_submit_request(pd, ctx, params, 613 KCF_RHNDL(KM_SLEEP)); 614 break; 615 616 case CRYPTO_HW_PROVIDER: 617 taskq = pd->pd_taskq; 618 619 /* 620 * Special case for CRYPTO_SYNCHRONOUS providers that 621 * never return a CRYPTO_QUEUED error. We skip any 622 * request allocation and call the SPI directly. 623 */ 624 if ((pd->pd_flags & CRYPTO_SYNCHRONOUS) && 625 EMPTY_TASKQ(taskq)) { 626 mp = &(pd->pd_percpu_bins[CPU_SEQID]); 627 KCF_PROV_JOB_HOLD(mp); 628 629 if (pd->pd_state == KCF_PROV_READY) { 630 error = common_submit_request(pd, ctx, 631 params, KCF_RHNDL(KM_SLEEP)); 632 KCF_PROV_JOB_RELE(mp); 633 ASSERT(error != CRYPTO_QUEUED); 634 break; 635 } 636 KCF_PROV_JOB_RELE(mp); 637 } 638 639 sreq = kmem_cache_alloc(kcf_sreq_cache, KM_SLEEP); 640 sreq->sn_state = REQ_ALLOCATED; 641 sreq->sn_rv = CRYPTO_FAILED; 642 sreq->sn_params = params; 643 644 /* 645 * Note that we do not need to hold the context 646 * for synchronous case as the context will never 647 * become invalid underneath us. We do not need to hold 648 * the provider here either as the caller has a hold. 649 */ 650 sreq->sn_context = kcf_ctx; 651 ASSERT(KCF_PROV_REFHELD(pd)); 652 sreq->sn_provider = pd; 653 654 ASSERT(taskq != NULL); 655 /* 656 * Call the SPI directly if the taskq is empty and the 657 * provider is not busy, else dispatch to the taskq. 658 * Calling directly is fine as this is the synchronous 659 * case. This is unlike the asynchronous case where we 660 * must always dispatch to the taskq. 661 */ 662 if (EMPTY_TASKQ(taskq) && 663 pd->pd_state == KCF_PROV_READY) { 664 process_req_hwp(sreq); 665 } else { 666 /* 667 * We can not tell from taskq_dispatch() return 668 * value if we exceeded maxalloc. Hence the 669 * check here. Since we are allowed to wait in 670 * the synchronous case, we wait for the taskq 671 * to become empty. 672 */ 673 if (taskq->tq_nalloc >= crypto_taskq_maxalloc) { 674 taskq_wait(taskq); 675 } 676 677 (void) taskq_dispatch(taskq, process_req_hwp, 678 sreq, TQ_SLEEP); 679 } 680 681 /* 682 * Wait for the notification to arrive, 683 * if the operation is not done yet. 684 * Bug# 4722589 will make the wait a cv_wait_sig(). 685 */ 686 mutex_enter(&sreq->sn_lock); 687 while (sreq->sn_state < REQ_DONE) 688 cv_wait(&sreq->sn_cv, &sreq->sn_lock); 689 mutex_exit(&sreq->sn_lock); 690 691 error = sreq->sn_rv; 692 kmem_cache_free(kcf_sreq_cache, sreq); 693 694 break; 695 696 default: 697 error = CRYPTO_FAILED; 698 break; 699 } 700 701 } else { /* Asynchronous cases */ 702 switch (pd->pd_prov_type) { 703 case CRYPTO_SW_PROVIDER: 704 if (!(crq->cr_flag & CRYPTO_ALWAYS_QUEUE)) { 705 /* 706 * This case has less overhead since there is 707 * no switching of context. 708 */ 709 error = common_submit_request(pd, ctx, params, 710 KCF_RHNDL(KM_NOSLEEP)); 711 } else { 712 /* 713 * CRYPTO_ALWAYS_QUEUE is set. We need to 714 * queue the request and return. 715 */ 716 areq = kcf_areqnode_alloc(pd, kcf_ctx, crq, 717 params, cont); 718 if (areq == NULL) 719 error = CRYPTO_HOST_MEMORY; 720 else { 721 if (!(crq->cr_flag 722 & CRYPTO_SKIP_REQID)) { 723 /* 724 * Set the request handle. This handle 725 * is used for any crypto_cancel_req(9f) 726 * calls from the consumer. We have to 727 * do this before dispatching the 728 * request. 729 */ 730 crq->cr_reqid = kcf_reqid_insert(areq); 731 } 732 733 error = kcf_disp_sw_request(areq); 734 /* 735 * There is an error processing this 736 * request. Remove the handle and 737 * release the request structure. 738 */ 739 if (error != CRYPTO_QUEUED) { 740 if (!(crq->cr_flag 741 & CRYPTO_SKIP_REQID)) 742 kcf_reqid_delete(areq); 743 KCF_AREQ_REFRELE(areq); 744 } 745 } 746 } 747 break; 748 749 case CRYPTO_HW_PROVIDER: 750 /* 751 * We need to queue the request and return. 752 */ 753 areq = kcf_areqnode_alloc(pd, kcf_ctx, crq, params, 754 cont); 755 if (areq == NULL) { 756 error = CRYPTO_HOST_MEMORY; 757 goto done; 758 } 759 760 taskq = pd->pd_taskq; 761 ASSERT(taskq != NULL); 762 /* 763 * We can not tell from taskq_dispatch() return 764 * value if we exceeded maxalloc. Hence the check 765 * here. 766 */ 767 if (taskq->tq_nalloc >= crypto_taskq_maxalloc) { 768 error = CRYPTO_BUSY; 769 KCF_AREQ_REFRELE(areq); 770 goto done; 771 } 772 773 if (!(crq->cr_flag & CRYPTO_SKIP_REQID)) { 774 /* 775 * Set the request handle. This handle is used 776 * for any crypto_cancel_req(9f) calls from the 777 * consumer. We have to do this before dispatching 778 * the request. 779 */ 780 crq->cr_reqid = kcf_reqid_insert(areq); 781 } 782 783 if (taskq_dispatch(taskq, 784 process_req_hwp, areq, TQ_NOSLEEP) == 785 (taskqid_t)0) { 786 error = CRYPTO_HOST_MEMORY; 787 if (!(crq->cr_flag & CRYPTO_SKIP_REQID)) 788 kcf_reqid_delete(areq); 789 KCF_AREQ_REFRELE(areq); 790 } else { 791 error = CRYPTO_QUEUED; 792 } 793 break; 794 795 default: 796 error = CRYPTO_FAILED; 797 break; 798 } 799 } 800 801 done: 802 return (error); 803 } 804 805 /* 806 * We're done with this framework context, so free it. Note that freeing 807 * framework context (kcf_context) frees the global context (crypto_ctx). 808 * 809 * The provider is responsible for freeing provider private context after a 810 * final or single operation and resetting the cc_provider_private field 811 * to NULL. It should do this before it notifies the framework of the 812 * completion. We still need to call KCF_PROV_FREE_CONTEXT to handle cases 813 * like crypto_cancel_ctx(9f). 814 */ 815 void 816 kcf_free_context(kcf_context_t *kcf_ctx) 817 { 818 kcf_provider_desc_t *pd = kcf_ctx->kc_prov_desc; 819 crypto_ctx_t *gctx = &kcf_ctx->kc_glbl_ctx; 820 kcf_context_t *kcf_secondctx = kcf_ctx->kc_secondctx; 821 kcf_prov_cpu_t *mp; 822 823 /* Release the second context, if any */ 824 825 if (kcf_secondctx != NULL) 826 KCF_CONTEXT_REFRELE(kcf_secondctx); 827 828 if (gctx->cc_provider_private != NULL) { 829 mutex_enter(&pd->pd_lock); 830 if (!KCF_IS_PROV_REMOVED(pd)) { 831 /* 832 * Increment the provider's internal refcnt so it 833 * doesn't unregister from the framework while 834 * we're calling the entry point. 835 */ 836 mp = &(pd->pd_percpu_bins[CPU_SEQID]); 837 KCF_PROV_JOB_HOLD(mp); 838 mutex_exit(&pd->pd_lock); 839 (void) KCF_PROV_FREE_CONTEXT(pd, gctx); 840 KCF_PROV_JOB_RELE(mp); 841 } else { 842 mutex_exit(&pd->pd_lock); 843 } 844 } 845 846 /* kcf_ctx->kc_prov_desc has a hold on pd */ 847 KCF_PROV_REFRELE(kcf_ctx->kc_prov_desc); 848 849 /* check if this context is shared with a software provider */ 850 if ((gctx->cc_flags & CRYPTO_INIT_OPSTATE) && 851 kcf_ctx->kc_sw_prov_desc != NULL) { 852 KCF_PROV_REFRELE(kcf_ctx->kc_sw_prov_desc); 853 } 854 855 kmem_cache_free(kcf_context_cache, kcf_ctx); 856 } 857 858 /* 859 * Free the request after releasing all the holds. 860 */ 861 void 862 kcf_free_req(kcf_areq_node_t *areq) 863 { 864 KCF_PROV_REFRELE(areq->an_provider); 865 if (areq->an_context != NULL) 866 KCF_CONTEXT_REFRELE(areq->an_context); 867 868 if (areq->an_tried_plist != NULL) 869 kcf_free_triedlist(areq->an_tried_plist); 870 kmem_cache_free(kcf_areq_cache, areq); 871 } 872 873 /* 874 * Utility routine to remove a request from the chain of requests 875 * hanging off a context. 876 */ 877 void 878 kcf_removereq_in_ctxchain(kcf_context_t *ictx, kcf_areq_node_t *areq) 879 { 880 kcf_areq_node_t *cur, *prev; 881 882 /* 883 * Get context lock, search for areq in the chain and remove it. 884 */ 885 ASSERT(ictx != NULL); 886 mutex_enter(&ictx->kc_in_use_lock); 887 prev = cur = ictx->kc_req_chain_first; 888 889 while (cur != NULL) { 890 if (cur == areq) { 891 if (prev == cur) { 892 if ((ictx->kc_req_chain_first = 893 cur->an_ctxchain_next) == NULL) 894 ictx->kc_req_chain_last = NULL; 895 } else { 896 if (cur == ictx->kc_req_chain_last) 897 ictx->kc_req_chain_last = prev; 898 prev->an_ctxchain_next = cur->an_ctxchain_next; 899 } 900 901 break; 902 } 903 prev = cur; 904 cur = cur->an_ctxchain_next; 905 } 906 mutex_exit(&ictx->kc_in_use_lock); 907 } 908 909 /* 910 * Remove the specified node from the global software queue. 911 * 912 * The caller must hold the queue lock and request lock (an_lock). 913 */ 914 void 915 kcf_remove_node(kcf_areq_node_t *node) 916 { 917 kcf_areq_node_t *nextp = node->an_next; 918 kcf_areq_node_t *prevp = node->an_prev; 919 920 ASSERT(mutex_owned(&gswq->gs_lock)); 921 922 if (nextp != NULL) 923 nextp->an_prev = prevp; 924 else 925 gswq->gs_last = prevp; 926 927 if (prevp != NULL) 928 prevp->an_next = nextp; 929 else 930 gswq->gs_first = nextp; 931 932 ASSERT(mutex_owned(&node->an_lock)); 933 node->an_state = REQ_CANCELED; 934 } 935 936 /* 937 * Remove and return the first node in the global software queue. 938 * 939 * The caller must hold the queue lock. 940 */ 941 static kcf_areq_node_t * 942 kcf_dequeue(void) 943 { 944 kcf_areq_node_t *tnode = NULL; 945 946 ASSERT(mutex_owned(&gswq->gs_lock)); 947 if ((tnode = gswq->gs_first) == NULL) { 948 return (NULL); 949 } else { 950 ASSERT(gswq->gs_first->an_prev == NULL); 951 gswq->gs_first = tnode->an_next; 952 if (tnode->an_next == NULL) 953 gswq->gs_last = NULL; 954 else 955 tnode->an_next->an_prev = NULL; 956 } 957 958 gswq->gs_njobs--; 959 return (tnode); 960 } 961 962 /* 963 * Add the request node to the end of the global software queue. 964 * 965 * The caller should not hold the queue lock. Returns 0 if the 966 * request is successfully queued. Returns CRYPTO_BUSY if the limit 967 * on the number of jobs is exceeded. 968 */ 969 static int 970 kcf_enqueue(kcf_areq_node_t *node) 971 { 972 kcf_areq_node_t *tnode; 973 974 mutex_enter(&gswq->gs_lock); 975 976 if (gswq->gs_njobs >= gswq->gs_maxjobs) { 977 mutex_exit(&gswq->gs_lock); 978 return (CRYPTO_BUSY); 979 } 980 981 if (gswq->gs_last == NULL) { 982 gswq->gs_first = gswq->gs_last = node; 983 } else { 984 ASSERT(gswq->gs_last->an_next == NULL); 985 tnode = gswq->gs_last; 986 tnode->an_next = node; 987 gswq->gs_last = node; 988 node->an_prev = tnode; 989 } 990 991 gswq->gs_njobs++; 992 993 /* an_lock not needed here as we hold gs_lock */ 994 node->an_state = REQ_WAITING; 995 996 mutex_exit(&gswq->gs_lock); 997 998 return (0); 999 } 1000 1001 /* 1002 * Function run by a thread from kcfpool to work on global software queue. 1003 */ 1004 void 1005 kcfpool_svc(void *arg) 1006 { 1007 _NOTE(ARGUNUSED(arg)); 1008 int error = 0; 1009 clock_t rv; 1010 clock_t timeout_val = drv_usectohz(kcf_idlethr_timeout); 1011 kcf_areq_node_t *req; 1012 kcf_context_t *ictx; 1013 kcf_provider_desc_t *pd; 1014 1015 KCF_ATOMIC_INCR(kcfpool->kp_threads); 1016 1017 for (;;) { 1018 mutex_enter(&gswq->gs_lock); 1019 1020 while ((req = kcf_dequeue()) == NULL) { 1021 KCF_ATOMIC_INCR(kcfpool->kp_idlethreads); 1022 rv = cv_reltimedwait(&gswq->gs_cv, 1023 &gswq->gs_lock, timeout_val, TR_CLOCK_TICK); 1024 KCF_ATOMIC_DECR(kcfpool->kp_idlethreads); 1025 1026 switch (rv) { 1027 case 0: 1028 case -1: 1029 /* 1030 * Woke up with no work to do. Check 1031 * if this thread should exit. We keep 1032 * at least kcf_minthreads. 1033 */ 1034 if (kcfpool->kp_threads > kcf_minthreads) { 1035 KCF_ATOMIC_DECR(kcfpool->kp_threads); 1036 mutex_exit(&gswq->gs_lock); 1037 return; 1038 } 1039 1040 /* Resume the wait for work. */ 1041 break; 1042 1043 default: 1044 /* 1045 * We are signaled to work on the queue. 1046 */ 1047 break; 1048 } 1049 } 1050 1051 mutex_exit(&gswq->gs_lock); 1052 1053 ictx = req->an_context; 1054 if (ictx == NULL) { /* Context-less operation */ 1055 pd = req->an_provider; 1056 error = common_submit_request(pd, NULL, 1057 &req->an_params, req); 1058 kcf_aop_done(req, error); 1059 continue; 1060 } 1061 1062 /* 1063 * We check if we can work on the request now. 1064 * Solaris does not guarantee any order on how the threads 1065 * are scheduled or how the waiters on a mutex are chosen. 1066 * So, we need to maintain our own order. 1067 * 1068 * is_my_turn is set to B_TRUE initially for a request when 1069 * it is enqueued and there are no other requests 1070 * for that context. Note that a thread sleeping on 1071 * an_turn_cv is not counted as an idle thread. This is 1072 * because we define an idle thread as one that sleeps on the 1073 * global queue waiting for new requests. 1074 */ 1075 mutex_enter(&req->an_lock); 1076 while (req->an_is_my_turn == B_FALSE) { 1077 KCF_ATOMIC_INCR(kcfpool->kp_blockedthreads); 1078 cv_wait(&req->an_turn_cv, &req->an_lock); 1079 KCF_ATOMIC_DECR(kcfpool->kp_blockedthreads); 1080 } 1081 1082 req->an_state = REQ_INPROGRESS; 1083 mutex_exit(&req->an_lock); 1084 1085 pd = ictx->kc_prov_desc; 1086 ASSERT(pd == req->an_provider); 1087 error = common_submit_request(pd, &ictx->kc_glbl_ctx, 1088 &req->an_params, req); 1089 1090 kcf_aop_done(req, error); 1091 } 1092 } 1093 1094 /* 1095 * kmem_cache_alloc constructor for sync request structure. 1096 */ 1097 /* ARGSUSED */ 1098 static int 1099 kcf_sreq_cache_constructor(void *buf, void *cdrarg, int kmflags) 1100 { 1101 kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)buf; 1102 1103 sreq->sn_type = CRYPTO_SYNCH; 1104 cv_init(&sreq->sn_cv, NULL, CV_DEFAULT, NULL); 1105 mutex_init(&sreq->sn_lock, NULL, MUTEX_DEFAULT, NULL); 1106 1107 return (0); 1108 } 1109 1110 /* ARGSUSED */ 1111 static void 1112 kcf_sreq_cache_destructor(void *buf, void *cdrarg) 1113 { 1114 kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)buf; 1115 1116 mutex_destroy(&sreq->sn_lock); 1117 cv_destroy(&sreq->sn_cv); 1118 } 1119 1120 /* 1121 * kmem_cache_alloc constructor for async request structure. 1122 */ 1123 /* ARGSUSED */ 1124 static int 1125 kcf_areq_cache_constructor(void *buf, void *cdrarg, int kmflags) 1126 { 1127 kcf_areq_node_t *areq = (kcf_areq_node_t *)buf; 1128 1129 areq->an_type = CRYPTO_ASYNCH; 1130 areq->an_refcnt = 0; 1131 mutex_init(&areq->an_lock, NULL, MUTEX_DEFAULT, NULL); 1132 cv_init(&areq->an_done, NULL, CV_DEFAULT, NULL); 1133 cv_init(&areq->an_turn_cv, NULL, CV_DEFAULT, NULL); 1134 1135 return (0); 1136 } 1137 1138 /* ARGSUSED */ 1139 static void 1140 kcf_areq_cache_destructor(void *buf, void *cdrarg) 1141 { 1142 kcf_areq_node_t *areq = (kcf_areq_node_t *)buf; 1143 1144 ASSERT(areq->an_refcnt == 0); 1145 mutex_destroy(&areq->an_lock); 1146 cv_destroy(&areq->an_done); 1147 cv_destroy(&areq->an_turn_cv); 1148 } 1149 1150 /* 1151 * kmem_cache_alloc constructor for kcf_context structure. 1152 */ 1153 /* ARGSUSED */ 1154 static int 1155 kcf_context_cache_constructor(void *buf, void *cdrarg, int kmflags) 1156 { 1157 kcf_context_t *kctx = (kcf_context_t *)buf; 1158 1159 kctx->kc_refcnt = 0; 1160 mutex_init(&kctx->kc_in_use_lock, NULL, MUTEX_DEFAULT, NULL); 1161 1162 return (0); 1163 } 1164 1165 /* ARGSUSED */ 1166 static void 1167 kcf_context_cache_destructor(void *buf, void *cdrarg) 1168 { 1169 kcf_context_t *kctx = (kcf_context_t *)buf; 1170 1171 ASSERT(kctx->kc_refcnt == 0); 1172 mutex_destroy(&kctx->kc_in_use_lock); 1173 } 1174 1175 /* 1176 * Creates and initializes all the structures needed by the framework. 1177 */ 1178 void 1179 kcf_sched_init(void) 1180 { 1181 int i; 1182 kcf_reqid_table_t *rt; 1183 1184 /* 1185 * Create all the kmem caches needed by the framework. We set the 1186 * align argument to 64, to get a slab aligned to 64-byte as well as 1187 * have the objects (cache_chunksize) to be a 64-byte multiple. 1188 * This helps to avoid false sharing as this is the size of the 1189 * CPU cache line. 1190 */ 1191 kcf_sreq_cache = kmem_cache_create("kcf_sreq_cache", 1192 sizeof (struct kcf_sreq_node), 64, kcf_sreq_cache_constructor, 1193 kcf_sreq_cache_destructor, NULL, NULL, NULL, 0); 1194 1195 kcf_areq_cache = kmem_cache_create("kcf_areq_cache", 1196 sizeof (struct kcf_areq_node), 64, kcf_areq_cache_constructor, 1197 kcf_areq_cache_destructor, NULL, NULL, NULL, 0); 1198 1199 kcf_context_cache = kmem_cache_create("kcf_context_cache", 1200 sizeof (struct kcf_context), 64, kcf_context_cache_constructor, 1201 kcf_context_cache_destructor, NULL, NULL, NULL, 0); 1202 1203 gswq = kmem_alloc(sizeof (kcf_global_swq_t), KM_SLEEP); 1204 1205 mutex_init(&gswq->gs_lock, NULL, MUTEX_DEFAULT, NULL); 1206 cv_init(&gswq->gs_cv, NULL, CV_DEFAULT, NULL); 1207 gswq->gs_njobs = 0; 1208 gswq->gs_maxjobs = kcf_maxthreads * crypto_taskq_maxalloc; 1209 gswq->gs_first = gswq->gs_last = NULL; 1210 1211 /* Initialize the global reqid table */ 1212 for (i = 0; i < REQID_TABLES; i++) { 1213 rt = kmem_zalloc(sizeof (kcf_reqid_table_t), KM_SLEEP); 1214 kcf_reqid_table[i] = rt; 1215 mutex_init(&rt->rt_lock, NULL, MUTEX_DEFAULT, NULL); 1216 rt->rt_curid = i; 1217 } 1218 1219 /* Allocate and initialize the thread pool */ 1220 kcfpool_alloc(); 1221 1222 /* Initialize the event notification list variables */ 1223 mutex_init(&ntfy_list_lock, NULL, MUTEX_DEFAULT, NULL); 1224 cv_init(&ntfy_list_cv, NULL, CV_DEFAULT, NULL); 1225 1226 /* Initialize the crypto_bufcall list variables */ 1227 mutex_init(&cbuf_list_lock, NULL, MUTEX_DEFAULT, NULL); 1228 cv_init(&cbuf_list_cv, NULL, CV_DEFAULT, NULL); 1229 1230 /* Create the kcf kstat */ 1231 kcf_misc_kstat = kstat_create("kcf", 0, "framework_stats", "crypto", 1232 KSTAT_TYPE_NAMED, sizeof (kcf_stats_t) / sizeof (kstat_named_t), 1233 KSTAT_FLAG_VIRTUAL); 1234 1235 if (kcf_misc_kstat != NULL) { 1236 kcf_misc_kstat->ks_data = &kcf_ksdata; 1237 kcf_misc_kstat->ks_update = kcf_misc_kstat_update; 1238 kstat_install(kcf_misc_kstat); 1239 } 1240 } 1241 1242 /* 1243 * This routine should only be called by drv/cryptoadm. 1244 * 1245 * kcf_sched_running flag isn't protected by a lock. But, we are safe because 1246 * the first thread ("cryptoadm refresh") calling this routine during 1247 * boot time completes before any other thread that can call this routine. 1248 */ 1249 void 1250 kcf_sched_start(void) 1251 { 1252 if (kcf_sched_running) 1253 return; 1254 1255 /* Start the background processing thread. */ 1256 (void) thread_create(NULL, 0, &crypto_bufcall_service, 0, 0, &p0, 1257 TS_RUN, minclsyspri); 1258 1259 kcf_sched_running = B_TRUE; 1260 } 1261 1262 /* 1263 * Signal the waiting sync client. 1264 */ 1265 void 1266 kcf_sop_done(kcf_sreq_node_t *sreq, int error) 1267 { 1268 mutex_enter(&sreq->sn_lock); 1269 sreq->sn_state = REQ_DONE; 1270 sreq->sn_rv = error; 1271 cv_signal(&sreq->sn_cv); 1272 mutex_exit(&sreq->sn_lock); 1273 } 1274 1275 /* 1276 * Callback the async client with the operation status. 1277 * We free the async request node and possibly the context. 1278 * We also handle any chain of requests hanging off of 1279 * the context. 1280 */ 1281 void 1282 kcf_aop_done(kcf_areq_node_t *areq, int error) 1283 { 1284 kcf_op_type_t optype; 1285 boolean_t skip_notify = B_FALSE; 1286 kcf_context_t *ictx; 1287 kcf_areq_node_t *nextreq; 1288 1289 /* 1290 * Handle recoverable errors. This has to be done first 1291 * before doing any thing else in this routine so that 1292 * we do not change the state of the request. 1293 */ 1294 if (error != CRYPTO_SUCCESS && IS_RECOVERABLE(error)) { 1295 /* 1296 * We try another provider, if one is available. Else 1297 * we continue with the failure notification to the 1298 * client. 1299 */ 1300 if (kcf_resubmit_request(areq) == CRYPTO_QUEUED) 1301 return; 1302 } 1303 1304 mutex_enter(&areq->an_lock); 1305 areq->an_state = REQ_DONE; 1306 mutex_exit(&areq->an_lock); 1307 1308 optype = (&areq->an_params)->rp_optype; 1309 if ((ictx = areq->an_context) != NULL) { 1310 /* 1311 * A request after it is removed from the request 1312 * queue, still stays on a chain of requests hanging 1313 * of its context structure. It needs to be removed 1314 * from this chain at this point. 1315 */ 1316 mutex_enter(&ictx->kc_in_use_lock); 1317 nextreq = areq->an_ctxchain_next; 1318 if (nextreq != NULL) { 1319 mutex_enter(&nextreq->an_lock); 1320 nextreq->an_is_my_turn = B_TRUE; 1321 cv_signal(&nextreq->an_turn_cv); 1322 mutex_exit(&nextreq->an_lock); 1323 } 1324 1325 ictx->kc_req_chain_first = nextreq; 1326 if (nextreq == NULL) 1327 ictx->kc_req_chain_last = NULL; 1328 mutex_exit(&ictx->kc_in_use_lock); 1329 1330 if (IS_SINGLE_OP(optype) || IS_FINAL_OP(optype)) { 1331 ASSERT(nextreq == NULL); 1332 KCF_CONTEXT_REFRELE(ictx); 1333 } else if (error != CRYPTO_SUCCESS && IS_INIT_OP(optype)) { 1334 /* 1335 * NOTE - We do not release the context in case of update 1336 * operations. We require the consumer to free it explicitly, 1337 * in case it wants to abandon an update operation. This is done 1338 * as there may be mechanisms in ECB mode that can continue 1339 * even if an operation on a block fails. 1340 */ 1341 KCF_CONTEXT_REFRELE(ictx); 1342 } 1343 } 1344 1345 /* Deal with the internal continuation to this request first */ 1346 1347 if (areq->an_isdual) { 1348 kcf_dual_req_t *next_arg; 1349 next_arg = (kcf_dual_req_t *)areq->an_reqarg.cr_callback_arg; 1350 next_arg->kr_areq = areq; 1351 KCF_AREQ_REFHOLD(areq); 1352 areq->an_isdual = B_FALSE; 1353 1354 NOTIFY_CLIENT(areq, error); 1355 return; 1356 } 1357 1358 /* 1359 * If CRYPTO_NOTIFY_OPDONE flag is set, we should notify 1360 * always. If this flag is clear, we skip the notification 1361 * provided there are no errors. We check this flag for only 1362 * init or update operations. It is ignored for single, final or 1363 * atomic operations. 1364 */ 1365 skip_notify = (IS_UPDATE_OP(optype) || IS_INIT_OP(optype)) && 1366 (!(areq->an_reqarg.cr_flag & CRYPTO_NOTIFY_OPDONE)) && 1367 (error == CRYPTO_SUCCESS); 1368 1369 if (!skip_notify) { 1370 NOTIFY_CLIENT(areq, error); 1371 } 1372 1373 if (!(areq->an_reqarg.cr_flag & CRYPTO_SKIP_REQID)) 1374 kcf_reqid_delete(areq); 1375 1376 KCF_AREQ_REFRELE(areq); 1377 } 1378 1379 /* 1380 * kcfpool thread spawner. This runs as a process that never exits. 1381 * Its a process so that the threads it owns can be manipulated via priocntl. 1382 */ 1383 static void 1384 kcfpoold(void *arg) 1385 { 1386 callb_cpr_t cprinfo; 1387 user_t *pu = PTOU(curproc); 1388 int cnt; 1389 clock_t timeout_val = drv_usectohz(kcf_idlethr_timeout); 1390 _NOTE(ARGUNUSED(arg)); 1391 1392 CALLB_CPR_INIT(&cprinfo, &kcfpool->kp_lock, 1393 callb_generic_cpr, "kcfpool"); 1394 1395 /* make our process "kcfpoold" */ 1396 (void) snprintf(pu->u_psargs, sizeof (pu->u_psargs), "kcfpoold"); 1397 (void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm)); 1398 1399 mutex_enter(&kcfpool->kp_lock); 1400 1401 /* 1402 * Go to sleep, waiting for the signaled flag. Note that as 1403 * we always do the same thing, and its always idempotent, we 1404 * don't even need to have a real condition to check against. 1405 */ 1406 for (;;) { 1407 int rv; 1408 1409 CALLB_CPR_SAFE_BEGIN(&cprinfo); 1410 rv = cv_reltimedwait(&kcfpool->kp_cv, 1411 &kcfpool->kp_lock, timeout_val, TR_CLOCK_TICK); 1412 CALLB_CPR_SAFE_END(&cprinfo, &kcfpool->kp_lock); 1413 1414 switch (rv) { 1415 case -1: 1416 /* Timed out. Recalculate the min/max threads */ 1417 compute_min_max_threads(); 1418 break; 1419 1420 default: 1421 /* Someone may be looking for a worker thread */ 1422 break; 1423 } 1424 1425 /* 1426 * We keep the number of running threads to be at 1427 * kcf_minthreads to reduce gs_lock contention. 1428 */ 1429 cnt = kcf_minthreads - 1430 (kcfpool->kp_threads - kcfpool->kp_blockedthreads); 1431 if (cnt > 0) { 1432 /* 1433 * The following ensures the number of threads in pool 1434 * does not exceed kcf_maxthreads. 1435 */ 1436 cnt = min(cnt, kcf_maxthreads - kcfpool->kp_threads); 1437 } 1438 1439 for (int i = 0; i < cnt; i++) { 1440 (void) lwp_kernel_create(curproc, 1441 kcfpool_svc, NULL, TS_RUN, curthread->t_pri); 1442 } 1443 } 1444 } 1445 1446 /* 1447 * Allocate the thread pool and initialize all the fields. 1448 */ 1449 static void 1450 kcfpool_alloc(void) 1451 { 1452 kcfpool = kmem_alloc(sizeof (kcf_pool_t), KM_SLEEP); 1453 1454 kcfpool->kp_threads = kcfpool->kp_idlethreads = 0; 1455 kcfpool->kp_blockedthreads = 0; 1456 1457 mutex_init(&kcfpool->kp_lock, NULL, MUTEX_DEFAULT, NULL); 1458 cv_init(&kcfpool->kp_cv, NULL, CV_DEFAULT, NULL); 1459 1460 kcf_idlethr_timeout = KCF_DEFAULT_THRTIMEOUT; 1461 1462 /* 1463 * Create the daemon thread. 1464 */ 1465 if (newproc(kcfpoold, NULL, syscid, minclsyspri, 1466 NULL, 0) != 0) { 1467 cmn_err(CE_PANIC, "unable to fork kcfpoold()"); 1468 } 1469 } 1470 1471 /* 1472 * This routine introduces a locking order for gswq->gs_lock followed 1473 * by cpu_lock. 1474 * This means that no consumer of the k-api should hold cpu_lock when calling 1475 * k-api routines. 1476 */ 1477 static void 1478 compute_min_max_threads(void) 1479 { 1480 mutex_enter(&gswq->gs_lock); 1481 mutex_enter(&cpu_lock); 1482 kcf_minthreads = curthread->t_cpupart->cp_ncpus; 1483 mutex_exit(&cpu_lock); 1484 kcf_maxthreads = kcf_thr_multiple * kcf_minthreads; 1485 gswq->gs_maxjobs = kcf_maxthreads * crypto_taskq_maxalloc; 1486 mutex_exit(&gswq->gs_lock); 1487 } 1488 1489 /* 1490 * Insert the async request in the hash table after assigning it 1491 * an ID. Returns the ID. 1492 * 1493 * The ID is used by the caller to pass as an argument to a 1494 * cancel_req() routine later. 1495 */ 1496 static crypto_req_id_t 1497 kcf_reqid_insert(kcf_areq_node_t *areq) 1498 { 1499 int indx; 1500 crypto_req_id_t id; 1501 kcf_areq_node_t *headp; 1502 kcf_reqid_table_t *rt = 1503 kcf_reqid_table[CPU->cpu_seqid & REQID_TABLE_MASK]; 1504 1505 mutex_enter(&rt->rt_lock); 1506 1507 rt->rt_curid = id = 1508 (rt->rt_curid - REQID_COUNTER_LOW) | REQID_COUNTER_HIGH; 1509 SET_REQID(areq, id); 1510 indx = REQID_HASH(id); 1511 headp = areq->an_idnext = rt->rt_idhash[indx]; 1512 areq->an_idprev = NULL; 1513 if (headp != NULL) 1514 headp->an_idprev = areq; 1515 1516 rt->rt_idhash[indx] = areq; 1517 mutex_exit(&rt->rt_lock); 1518 1519 return (id); 1520 } 1521 1522 /* 1523 * Delete the async request from the hash table. 1524 */ 1525 static void 1526 kcf_reqid_delete(kcf_areq_node_t *areq) 1527 { 1528 int indx; 1529 kcf_areq_node_t *nextp, *prevp; 1530 crypto_req_id_t id = GET_REQID(areq); 1531 kcf_reqid_table_t *rt; 1532 1533 rt = kcf_reqid_table[id & REQID_TABLE_MASK]; 1534 indx = REQID_HASH(id); 1535 1536 mutex_enter(&rt->rt_lock); 1537 1538 nextp = areq->an_idnext; 1539 prevp = areq->an_idprev; 1540 if (nextp != NULL) 1541 nextp->an_idprev = prevp; 1542 if (prevp != NULL) 1543 prevp->an_idnext = nextp; 1544 else 1545 rt->rt_idhash[indx] = nextp; 1546 1547 SET_REQID(areq, 0); 1548 cv_broadcast(&areq->an_done); 1549 1550 mutex_exit(&rt->rt_lock); 1551 } 1552 1553 /* 1554 * Cancel a single asynchronous request. 1555 * 1556 * We guarantee that no problems will result from calling 1557 * crypto_cancel_req() for a request which is either running, or 1558 * has already completed. We remove the request from any queues 1559 * if it is possible. We wait for request completion if the 1560 * request is dispatched to a provider. 1561 * 1562 * Calling context: 1563 * Can be called from user context only. 1564 * 1565 * NOTE: We acquire the following locks in this routine (in order): 1566 * - rt_lock (kcf_reqid_table_t) 1567 * - gswq->gs_lock 1568 * - areq->an_lock 1569 * - ictx->kc_in_use_lock (from kcf_removereq_in_ctxchain()) 1570 * 1571 * This locking order MUST be maintained in code every where else. 1572 */ 1573 void 1574 crypto_cancel_req(crypto_req_id_t id) 1575 { 1576 int indx; 1577 kcf_areq_node_t *areq; 1578 kcf_provider_desc_t *pd; 1579 kcf_context_t *ictx; 1580 kcf_reqid_table_t *rt; 1581 1582 rt = kcf_reqid_table[id & REQID_TABLE_MASK]; 1583 indx = REQID_HASH(id); 1584 1585 mutex_enter(&rt->rt_lock); 1586 for (areq = rt->rt_idhash[indx]; areq; areq = areq->an_idnext) { 1587 if (GET_REQID(areq) == id) { 1588 /* 1589 * We found the request. It is either still waiting 1590 * in the framework queues or running at the provider. 1591 */ 1592 pd = areq->an_provider; 1593 ASSERT(pd != NULL); 1594 1595 switch (pd->pd_prov_type) { 1596 case CRYPTO_SW_PROVIDER: 1597 mutex_enter(&gswq->gs_lock); 1598 mutex_enter(&areq->an_lock); 1599 1600 /* This request can be safely canceled. */ 1601 if (areq->an_state <= REQ_WAITING) { 1602 /* Remove from gswq, global software queue. */ 1603 kcf_remove_node(areq); 1604 if ((ictx = areq->an_context) != NULL) 1605 kcf_removereq_in_ctxchain(ictx, areq); 1606 1607 mutex_exit(&areq->an_lock); 1608 mutex_exit(&gswq->gs_lock); 1609 mutex_exit(&rt->rt_lock); 1610 1611 /* Remove areq from hash table and free it. */ 1612 kcf_reqid_delete(areq); 1613 KCF_AREQ_REFRELE(areq); 1614 return; 1615 } 1616 1617 mutex_exit(&areq->an_lock); 1618 mutex_exit(&gswq->gs_lock); 1619 break; 1620 1621 case CRYPTO_HW_PROVIDER: 1622 /* 1623 * There is no interface to remove an entry 1624 * once it is on the taskq. So, we do not do 1625 * any thing for a hardware provider. 1626 */ 1627 break; 1628 } 1629 1630 /* 1631 * The request is running. Wait for the request completion 1632 * to notify us. 1633 */ 1634 KCF_AREQ_REFHOLD(areq); 1635 while (GET_REQID(areq) == id) 1636 cv_wait(&areq->an_done, &rt->rt_lock); 1637 KCF_AREQ_REFRELE(areq); 1638 break; 1639 } 1640 } 1641 1642 mutex_exit(&rt->rt_lock); 1643 } 1644 1645 /* 1646 * Cancel all asynchronous requests associated with the 1647 * passed in crypto context and free it. 1648 * 1649 * A client SHOULD NOT call this routine after calling a crypto_*_final 1650 * routine. This routine is called only during intermediate operations. 1651 * The client should not use the crypto context after this function returns 1652 * since we destroy it. 1653 * 1654 * Calling context: 1655 * Can be called from user context only. 1656 */ 1657 void 1658 crypto_cancel_ctx(crypto_context_t ctx) 1659 { 1660 kcf_context_t *ictx; 1661 kcf_areq_node_t *areq; 1662 1663 if (ctx == NULL) 1664 return; 1665 1666 ictx = (kcf_context_t *)((crypto_ctx_t *)ctx)->cc_framework_private; 1667 1668 mutex_enter(&ictx->kc_in_use_lock); 1669 1670 /* Walk the chain and cancel each request */ 1671 while ((areq = ictx->kc_req_chain_first) != NULL) { 1672 /* 1673 * We have to drop the lock here as we may have 1674 * to wait for request completion. We hold the 1675 * request before dropping the lock though, so that it 1676 * won't be freed underneath us. 1677 */ 1678 KCF_AREQ_REFHOLD(areq); 1679 mutex_exit(&ictx->kc_in_use_lock); 1680 1681 crypto_cancel_req(GET_REQID(areq)); 1682 KCF_AREQ_REFRELE(areq); 1683 1684 mutex_enter(&ictx->kc_in_use_lock); 1685 } 1686 1687 mutex_exit(&ictx->kc_in_use_lock); 1688 KCF_CONTEXT_REFRELE(ictx); 1689 } 1690 1691 /* 1692 * Update kstats. 1693 */ 1694 static int 1695 kcf_misc_kstat_update(kstat_t *ksp, int rw) 1696 { 1697 kcf_stats_t *ks_data; 1698 1699 if (rw == KSTAT_WRITE) 1700 return (EACCES); 1701 1702 ks_data = ksp->ks_data; 1703 1704 ks_data->ks_thrs_in_pool.value.ui32 = kcfpool->kp_threads; 1705 ks_data->ks_idle_thrs.value.ui32 = kcfpool->kp_idlethreads; 1706 ks_data->ks_minthrs.value.ui32 = kcf_minthreads; 1707 ks_data->ks_maxthrs.value.ui32 = kcf_maxthreads; 1708 ks_data->ks_swq_njobs.value.ui32 = gswq->gs_njobs; 1709 ks_data->ks_swq_maxjobs.value.ui32 = gswq->gs_maxjobs; 1710 ks_data->ks_taskq_threads.value.ui32 = crypto_taskq_threads; 1711 ks_data->ks_taskq_minalloc.value.ui32 = crypto_taskq_minalloc; 1712 ks_data->ks_taskq_maxalloc.value.ui32 = crypto_taskq_maxalloc; 1713 1714 return (0); 1715 } 1716 1717 /* 1718 * Allocate and initiatize a kcf_dual_req, used for saving the arguments of 1719 * a dual operation or an atomic operation that has to be internally 1720 * simulated with multiple single steps. 1721 * crq determines the memory allocation flags. 1722 */ 1723 1724 kcf_dual_req_t * 1725 kcf_alloc_req(crypto_call_req_t *crq) 1726 { 1727 kcf_dual_req_t *kcr; 1728 1729 kcr = kmem_alloc(sizeof (kcf_dual_req_t), KCF_KMFLAG(crq)); 1730 1731 if (kcr == NULL) 1732 return (NULL); 1733 1734 /* Copy the whole crypto_call_req struct, as it isn't persistent */ 1735 if (crq != NULL) 1736 kcr->kr_callreq = *crq; 1737 else 1738 bzero(&(kcr->kr_callreq), sizeof (crypto_call_req_t)); 1739 kcr->kr_areq = NULL; 1740 kcr->kr_saveoffset = 0; 1741 kcr->kr_savelen = 0; 1742 1743 return (kcr); 1744 } 1745 1746 /* 1747 * Callback routine for the next part of a simulated dual part. 1748 * Schedules the next step. 1749 * 1750 * This routine can be called from interrupt context. 1751 */ 1752 void 1753 kcf_next_req(void *next_req_arg, int status) 1754 { 1755 kcf_dual_req_t *next_req = (kcf_dual_req_t *)next_req_arg; 1756 kcf_req_params_t *params = &(next_req->kr_params); 1757 kcf_areq_node_t *areq = next_req->kr_areq; 1758 int error = status; 1759 kcf_provider_desc_t *pd; 1760 crypto_dual_data_t *ct; 1761 1762 /* Stop the processing if an error occurred at this step */ 1763 if (error != CRYPTO_SUCCESS) { 1764 out: 1765 areq->an_reqarg = next_req->kr_callreq; 1766 KCF_AREQ_REFRELE(areq); 1767 kmem_free(next_req, sizeof (kcf_dual_req_t)); 1768 areq->an_isdual = B_FALSE; 1769 kcf_aop_done(areq, error); 1770 return; 1771 } 1772 1773 switch (params->rp_opgrp) { 1774 case KCF_OG_MAC: { 1775 1776 /* 1777 * The next req is submitted with the same reqid as the 1778 * first part. The consumer only got back that reqid, and 1779 * should still be able to cancel the operation during its 1780 * second step. 1781 */ 1782 kcf_mac_ops_params_t *mops = &(params->rp_u.mac_params); 1783 crypto_ctx_template_t mac_tmpl; 1784 kcf_mech_entry_t *me; 1785 1786 ct = (crypto_dual_data_t *)mops->mo_data; 1787 mac_tmpl = (crypto_ctx_template_t)mops->mo_templ; 1788 1789 /* No expected recoverable failures, so no retry list */ 1790 pd = kcf_get_mech_provider(mops->mo_framework_mechtype, NULL, 1791 &me, &error, NULL, CRYPTO_FG_MAC_ATOMIC, ct->dd_len2); 1792 1793 if (pd == NULL) { 1794 error = CRYPTO_MECH_NOT_SUPPORTED; 1795 goto out; 1796 } 1797 /* Validate the MAC context template here */ 1798 if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) && 1799 (mac_tmpl != NULL)) { 1800 kcf_ctx_template_t *ctx_mac_tmpl; 1801 1802 ctx_mac_tmpl = (kcf_ctx_template_t *)mac_tmpl; 1803 1804 if (ctx_mac_tmpl->ct_generation != me->me_gen_swprov) { 1805 KCF_PROV_REFRELE(pd); 1806 error = CRYPTO_OLD_CTX_TEMPLATE; 1807 goto out; 1808 } 1809 mops->mo_templ = ctx_mac_tmpl->ct_prov_tmpl; 1810 } 1811 1812 break; 1813 } 1814 case KCF_OG_DECRYPT: { 1815 kcf_decrypt_ops_params_t *dcrops = 1816 &(params->rp_u.decrypt_params); 1817 1818 ct = (crypto_dual_data_t *)dcrops->dop_ciphertext; 1819 /* No expected recoverable failures, so no retry list */ 1820 pd = kcf_get_mech_provider(dcrops->dop_framework_mechtype, 1821 NULL, NULL, &error, NULL, CRYPTO_FG_DECRYPT_ATOMIC, 1822 ct->dd_len1); 1823 1824 if (pd == NULL) { 1825 error = CRYPTO_MECH_NOT_SUPPORTED; 1826 goto out; 1827 } 1828 break; 1829 } 1830 } 1831 1832 /* The second step uses len2 and offset2 of the dual_data */ 1833 next_req->kr_saveoffset = ct->dd_offset1; 1834 next_req->kr_savelen = ct->dd_len1; 1835 ct->dd_offset1 = ct->dd_offset2; 1836 ct->dd_len1 = ct->dd_len2; 1837 1838 areq->an_reqarg.cr_flag = 0; 1839 1840 areq->an_reqarg.cr_callback_func = kcf_last_req; 1841 areq->an_reqarg.cr_callback_arg = next_req; 1842 areq->an_isdual = B_TRUE; 1843 1844 /* 1845 * We would like to call kcf_submit_request() here. But, 1846 * that is not possible as that routine allocates a new 1847 * kcf_areq_node_t request structure, while we need to 1848 * reuse the existing request structure. 1849 */ 1850 switch (pd->pd_prov_type) { 1851 case CRYPTO_SW_PROVIDER: 1852 error = common_submit_request(pd, NULL, params, 1853 KCF_RHNDL(KM_NOSLEEP)); 1854 break; 1855 1856 case CRYPTO_HW_PROVIDER: { 1857 kcf_provider_desc_t *old_pd; 1858 taskq_t *taskq = pd->pd_taskq; 1859 1860 /* 1861 * Set the params for the second step in the 1862 * dual-ops. 1863 */ 1864 areq->an_params = *params; 1865 old_pd = areq->an_provider; 1866 KCF_PROV_REFRELE(old_pd); 1867 KCF_PROV_REFHOLD(pd); 1868 areq->an_provider = pd; 1869 1870 /* 1871 * Note that we have to do a taskq_dispatch() 1872 * here as we may be in interrupt context. 1873 */ 1874 if (taskq_dispatch(taskq, process_req_hwp, areq, 1875 TQ_NOSLEEP) == (taskqid_t)0) { 1876 error = CRYPTO_HOST_MEMORY; 1877 } else { 1878 error = CRYPTO_QUEUED; 1879 } 1880 break; 1881 } 1882 } 1883 1884 /* 1885 * We have to release the holds on the request and the provider 1886 * in all cases. 1887 */ 1888 KCF_AREQ_REFRELE(areq); 1889 KCF_PROV_REFRELE(pd); 1890 1891 if (error != CRYPTO_QUEUED) { 1892 /* restore, clean up, and invoke the client's callback */ 1893 1894 ct->dd_offset1 = next_req->kr_saveoffset; 1895 ct->dd_len1 = next_req->kr_savelen; 1896 areq->an_reqarg = next_req->kr_callreq; 1897 kmem_free(next_req, sizeof (kcf_dual_req_t)); 1898 areq->an_isdual = B_FALSE; 1899 kcf_aop_done(areq, error); 1900 } 1901 } 1902 1903 /* 1904 * Last part of an emulated dual operation. 1905 * Clean up and restore ... 1906 */ 1907 void 1908 kcf_last_req(void *last_req_arg, int status) 1909 { 1910 kcf_dual_req_t *last_req = (kcf_dual_req_t *)last_req_arg; 1911 1912 kcf_req_params_t *params = &(last_req->kr_params); 1913 kcf_areq_node_t *areq = last_req->kr_areq; 1914 crypto_dual_data_t *ct; 1915 1916 switch (params->rp_opgrp) { 1917 case KCF_OG_MAC: { 1918 kcf_mac_ops_params_t *mops = &(params->rp_u.mac_params); 1919 1920 ct = (crypto_dual_data_t *)mops->mo_data; 1921 break; 1922 } 1923 case KCF_OG_DECRYPT: { 1924 kcf_decrypt_ops_params_t *dcrops = 1925 &(params->rp_u.decrypt_params); 1926 1927 ct = (crypto_dual_data_t *)dcrops->dop_ciphertext; 1928 break; 1929 } 1930 } 1931 ct->dd_offset1 = last_req->kr_saveoffset; 1932 ct->dd_len1 = last_req->kr_savelen; 1933 1934 /* The submitter used kcf_last_req as its callback */ 1935 1936 if (areq == NULL) { 1937 crypto_call_req_t *cr = &last_req->kr_callreq; 1938 1939 (*(cr->cr_callback_func))(cr->cr_callback_arg, status); 1940 kmem_free(last_req, sizeof (kcf_dual_req_t)); 1941 return; 1942 } 1943 areq->an_reqarg = last_req->kr_callreq; 1944 KCF_AREQ_REFRELE(areq); 1945 kmem_free(last_req, sizeof (kcf_dual_req_t)); 1946 areq->an_isdual = B_FALSE; 1947 kcf_aop_done(areq, status); 1948 } 1949