1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 28 /* 29 * Deimos - cryptographic acceleration based upon Broadcom 582x. 30 */ 31 32 #include <sys/types.h> 33 #include <sys/modctl.h> 34 #include <sys/conf.h> 35 #include <sys/devops.h> 36 #include <sys/ddi.h> 37 #include <sys/sunddi.h> 38 #include <sys/cmn_err.h> 39 #include <sys/varargs.h> 40 #include <sys/file.h> 41 #include <sys/stat.h> 42 #include <sys/kmem.h> 43 #include <sys/ioccom.h> 44 #include <sys/open.h> 45 #include <sys/cred.h> 46 #include <sys/kstat.h> 47 #include <sys/strsun.h> 48 #include <sys/note.h> 49 #include <sys/crypto/common.h> 50 #include <sys/crypto/spi.h> 51 #include <sys/ddifm.h> 52 #include <sys/fm/protocol.h> 53 #include <sys/fm/util.h> 54 #include <sys/fm/io/ddi.h> 55 #include <sys/crypto/dca.h> 56 57 /* 58 * Core Deimos driver. 59 */ 60 61 static void dca_enlist2(dca_listnode_t *, dca_listnode_t *, 62 kmutex_t *); 63 static void dca_rmlist2(dca_listnode_t *node, kmutex_t *); 64 static dca_listnode_t *dca_delist2(dca_listnode_t *q, kmutex_t *); 65 static void dca_free_context_list(dca_t *dca); 66 static int dca_free_context_low(crypto_ctx_t *ctx); 67 static int dca_attach(dev_info_t *, ddi_attach_cmd_t); 68 static int dca_detach(dev_info_t *, ddi_detach_cmd_t); 69 static int dca_suspend(dca_t *); 70 static int dca_resume(dca_t *); 71 static int dca_init(dca_t *); 72 static int dca_reset(dca_t *, int); 73 static int dca_initworklist(dca_t *, dca_worklist_t *); 74 static void dca_uninit(dca_t *); 75 static void dca_initq(dca_listnode_t *); 76 static void dca_enqueue(dca_listnode_t *, dca_listnode_t *); 77 static dca_listnode_t *dca_dequeue(dca_listnode_t *); 78 static dca_listnode_t *dca_unqueue(dca_listnode_t *); 79 static dca_request_t *dca_newreq(dca_t *); 80 static dca_work_t *dca_getwork(dca_t *, int); 81 static void dca_freework(dca_work_t *); 82 static dca_work_t *dca_newwork(dca_t *); 83 static void dca_destroywork(dca_work_t *); 84 static void dca_schedule(dca_t *, int); 85 static void dca_reclaim(dca_t *, int); 86 static uint_t dca_intr(char *); 87 static void dca_failure(dca_t *, ddi_fault_location_t, 88 dca_fma_eclass_t index, uint64_t, int, char *, ...); 89 static void dca_jobtimeout(void *); 90 static int dca_drain(dca_t *); 91 static void dca_undrain(dca_t *); 92 static void dca_rejectjobs(dca_t *); 93 94 #ifdef SCHEDDELAY 95 static void dca_schedtimeout(void *); 96 #endif 97 98 /* 99 * We want these inlined for performance. 100 */ 101 #ifndef DEBUG 102 #pragma inline(dca_freereq, dca_getreq, dca_freework, dca_getwork) 103 #pragma inline(dca_enqueue, dca_dequeue, dca_rmqueue, dca_done) 104 #pragma inline(dca_reverse, dca_length) 105 #endif 106 107 /* 108 * Device operations. 109 */ 110 static struct dev_ops devops = { 111 DEVO_REV, /* devo_rev */ 112 0, /* devo_refcnt */ 113 nodev, /* devo_getinfo */ 114 nulldev, /* devo_identify */ 115 nulldev, /* devo_probe */ 116 dca_attach, /* devo_attach */ 117 dca_detach, /* devo_detach */ 118 nodev, /* devo_reset */ 119 NULL, /* devo_cb_ops */ 120 NULL, /* devo_bus_ops */ 121 ddi_power, /* devo_power */ 122 ddi_quiesce_not_supported, /* devo_quiesce */ 123 }; 124 125 #define IDENT "PCI Crypto Accelerator" 126 #define IDENT_SYM "Crypto Accel Sym 2.0" 127 #define IDENT_ASYM "Crypto Accel Asym 2.0" 128 129 /* Space-padded, will be filled in dynamically during registration */ 130 #define IDENT3 "PCI Crypto Accelerator Mod 2.0" 131 132 #define VENDOR "Sun Microsystems, Inc." 133 134 #define STALETIME (30 * SECOND) 135 136 #define crypto_prov_notify crypto_provider_notification 137 /* A 28 char function name doesn't leave much line space */ 138 139 /* 140 * Module linkage. 141 */ 142 static struct modldrv modldrv = { 143 &mod_driverops, /* drv_modops */ 144 IDENT, /* drv_linkinfo */ 145 &devops, /* drv_dev_ops */ 146 }; 147 148 extern struct mod_ops mod_cryptoops; 149 150 static struct modlcrypto modlcrypto = { 151 &mod_cryptoops, 152 IDENT3 153 }; 154 155 static struct modlinkage modlinkage = { 156 MODREV_1, /* ml_rev */ 157 &modldrv, /* ml_linkage */ 158 &modlcrypto, 159 NULL 160 }; 161 162 /* 163 * CSPI information (entry points, provider info, etc.) 164 */ 165 166 /* Mechanisms for the symmetric cipher provider */ 167 static crypto_mech_info_t dca_mech_info_tab1[] = { 168 /* DES-CBC */ 169 {SUN_CKM_DES_CBC, DES_CBC_MECH_INFO_TYPE, 170 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | 171 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC, 172 DES_KEY_LEN, DES_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES}, 173 /* 3DES-CBC */ 174 {SUN_CKM_DES3_CBC, DES3_CBC_MECH_INFO_TYPE, 175 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | 176 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC, 177 DES3_MIN_KEY_LEN, DES3_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES} 178 }; 179 180 /* Mechanisms for the asymmetric cipher provider */ 181 static crypto_mech_info_t dca_mech_info_tab2[] = { 182 /* DSA */ 183 {SUN_CKM_DSA, DSA_MECH_INFO_TYPE, 184 CRYPTO_FG_SIGN | CRYPTO_FG_VERIFY | 185 CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_VERIFY_ATOMIC, 186 CRYPTO_BYTES2BITS(DSA_MIN_KEY_LEN), 187 CRYPTO_BYTES2BITS(DSA_MAX_KEY_LEN), 188 CRYPTO_KEYSIZE_UNIT_IN_BITS}, 189 190 /* RSA */ 191 {SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE, 192 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN | 193 CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY | 194 CRYPTO_FG_VERIFY_RECOVER | 195 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC | 196 CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC | 197 CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC, 198 CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN), 199 CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN), 200 CRYPTO_KEYSIZE_UNIT_IN_BITS}, 201 {SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE, 202 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN | 203 CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY | 204 CRYPTO_FG_VERIFY_RECOVER | 205 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC | 206 CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC | 207 CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC, 208 CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN), 209 CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN), 210 CRYPTO_KEYSIZE_UNIT_IN_BITS} 211 }; 212 213 static void dca_provider_status(crypto_provider_handle_t, uint_t *); 214 215 static crypto_control_ops_t dca_control_ops = { 216 dca_provider_status 217 }; 218 219 static int dca_encrypt_init(crypto_ctx_t *, crypto_mechanism_t *, 220 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 221 static int dca_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, 222 crypto_req_handle_t); 223 static int dca_encrypt_update(crypto_ctx_t *, crypto_data_t *, 224 crypto_data_t *, crypto_req_handle_t); 225 static int dca_encrypt_final(crypto_ctx_t *, crypto_data_t *, 226 crypto_req_handle_t); 227 static int dca_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t, 228 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, 229 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 230 231 static int dca_decrypt_init(crypto_ctx_t *, crypto_mechanism_t *, 232 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 233 static int dca_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, 234 crypto_req_handle_t); 235 static int dca_decrypt_update(crypto_ctx_t *, crypto_data_t *, 236 crypto_data_t *, crypto_req_handle_t); 237 static int dca_decrypt_final(crypto_ctx_t *, crypto_data_t *, 238 crypto_req_handle_t); 239 static int dca_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t, 240 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, 241 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 242 243 static crypto_cipher_ops_t dca_cipher_ops = { 244 dca_encrypt_init, 245 dca_encrypt, 246 dca_encrypt_update, 247 dca_encrypt_final, 248 dca_encrypt_atomic, 249 dca_decrypt_init, 250 dca_decrypt, 251 dca_decrypt_update, 252 dca_decrypt_final, 253 dca_decrypt_atomic 254 }; 255 256 static int dca_sign_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, 257 crypto_spi_ctx_template_t, crypto_req_handle_t); 258 static int dca_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, 259 crypto_req_handle_t); 260 static int dca_sign_update(crypto_ctx_t *, crypto_data_t *, 261 crypto_req_handle_t); 262 static int dca_sign_final(crypto_ctx_t *, crypto_data_t *, 263 crypto_req_handle_t); 264 static int dca_sign_atomic(crypto_provider_handle_t, crypto_session_id_t, 265 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, 266 crypto_spi_ctx_template_t, crypto_req_handle_t); 267 static int dca_sign_recover_init(crypto_ctx_t *, crypto_mechanism_t *, 268 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 269 static int dca_sign_recover(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, 270 crypto_req_handle_t); 271 static int dca_sign_recover_atomic(crypto_provider_handle_t, 272 crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, 273 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 274 275 static crypto_sign_ops_t dca_sign_ops = { 276 dca_sign_init, 277 dca_sign, 278 dca_sign_update, 279 dca_sign_final, 280 dca_sign_atomic, 281 dca_sign_recover_init, 282 dca_sign_recover, 283 dca_sign_recover_atomic 284 }; 285 286 static int dca_verify_init(crypto_ctx_t *, crypto_mechanism_t *, 287 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 288 static int dca_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, 289 crypto_req_handle_t); 290 static int dca_verify_update(crypto_ctx_t *, crypto_data_t *, 291 crypto_req_handle_t); 292 static int dca_verify_final(crypto_ctx_t *, crypto_data_t *, 293 crypto_req_handle_t); 294 static int dca_verify_atomic(crypto_provider_handle_t, crypto_session_id_t, 295 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, 296 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 297 static int dca_verify_recover_init(crypto_ctx_t *, crypto_mechanism_t *, 298 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 299 static int dca_verify_recover(crypto_ctx_t *, crypto_data_t *, 300 crypto_data_t *, crypto_req_handle_t); 301 static int dca_verify_recover_atomic(crypto_provider_handle_t, 302 crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, 303 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 304 305 static crypto_verify_ops_t dca_verify_ops = { 306 dca_verify_init, 307 dca_verify, 308 dca_verify_update, 309 dca_verify_final, 310 dca_verify_atomic, 311 dca_verify_recover_init, 312 dca_verify_recover, 313 dca_verify_recover_atomic 314 }; 315 316 static int dca_generate_random(crypto_provider_handle_t, crypto_session_id_t, 317 uchar_t *, size_t, crypto_req_handle_t); 318 319 static crypto_random_number_ops_t dca_random_number_ops = { 320 NULL, 321 dca_generate_random 322 }; 323 324 static int ext_info_sym(crypto_provider_handle_t prov, 325 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq); 326 static int ext_info_asym(crypto_provider_handle_t prov, 327 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq); 328 static int ext_info_base(crypto_provider_handle_t prov, 329 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id); 330 331 static crypto_provider_management_ops_t dca_provmanage_ops_1 = { 332 ext_info_sym, /* ext_info */ 333 NULL, /* init_token */ 334 NULL, /* init_pin */ 335 NULL /* set_pin */ 336 }; 337 338 static crypto_provider_management_ops_t dca_provmanage_ops_2 = { 339 ext_info_asym, /* ext_info */ 340 NULL, /* init_token */ 341 NULL, /* init_pin */ 342 NULL /* set_pin */ 343 }; 344 345 int dca_free_context(crypto_ctx_t *); 346 347 static crypto_ctx_ops_t dca_ctx_ops = { 348 NULL, 349 dca_free_context 350 }; 351 352 /* Operations for the symmetric cipher provider */ 353 static crypto_ops_t dca_crypto_ops1 = { 354 &dca_control_ops, 355 NULL, /* digest_ops */ 356 &dca_cipher_ops, 357 NULL, /* mac_ops */ 358 NULL, /* sign_ops */ 359 NULL, /* verify_ops */ 360 NULL, /* dual_ops */ 361 NULL, /* cipher_mac_ops */ 362 NULL, /* random_number_ops */ 363 NULL, /* session_ops */ 364 NULL, /* object_ops */ 365 NULL, /* key_ops */ 366 &dca_provmanage_ops_1, /* management_ops */ 367 &dca_ctx_ops 368 }; 369 370 /* Operations for the asymmetric cipher provider */ 371 static crypto_ops_t dca_crypto_ops2 = { 372 &dca_control_ops, 373 NULL, /* digest_ops */ 374 &dca_cipher_ops, 375 NULL, /* mac_ops */ 376 &dca_sign_ops, 377 &dca_verify_ops, 378 NULL, /* dual_ops */ 379 NULL, /* cipher_mac_ops */ 380 &dca_random_number_ops, 381 NULL, /* session_ops */ 382 NULL, /* object_ops */ 383 NULL, /* key_ops */ 384 &dca_provmanage_ops_2, /* management_ops */ 385 &dca_ctx_ops 386 }; 387 388 /* Provider information for the symmetric cipher provider */ 389 static crypto_provider_info_t dca_prov_info1 = { 390 CRYPTO_SPI_VERSION_1, 391 NULL, /* pi_provider_description */ 392 CRYPTO_HW_PROVIDER, 393 NULL, /* pi_provider_dev */ 394 NULL, /* pi_provider_handle */ 395 &dca_crypto_ops1, 396 sizeof (dca_mech_info_tab1)/sizeof (crypto_mech_info_t), 397 dca_mech_info_tab1, 398 0, /* pi_logical_provider_count */ 399 NULL /* pi_logical_providers */ 400 }; 401 402 /* Provider information for the asymmetric cipher provider */ 403 static crypto_provider_info_t dca_prov_info2 = { 404 CRYPTO_SPI_VERSION_1, 405 NULL, /* pi_provider_description */ 406 CRYPTO_HW_PROVIDER, 407 NULL, /* pi_provider_dev */ 408 NULL, /* pi_provider_handle */ 409 &dca_crypto_ops2, 410 sizeof (dca_mech_info_tab2)/sizeof (crypto_mech_info_t), 411 dca_mech_info_tab2, 412 0, /* pi_logical_provider_count */ 413 NULL /* pi_logical_providers */ 414 }; 415 416 /* Convenience macros */ 417 #define DCA_SOFTC_FROM_CTX(ctx) ((dca_t *)(ctx)->cc_provider) 418 #define DCA_MECH_FROM_CTX(ctx) \ 419 (((dca_request_t *)(ctx)->cc_provider_private)->dr_ctx.ctx_cm_type) 420 421 static int dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset, 422 caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags, 423 dca_chain_t *head, int *n_chain); 424 static uint64_t dca_ena(uint64_t ena); 425 static caddr_t dca_bufdaddr_out(crypto_data_t *data); 426 static char *dca_fma_eclass_string(char *model, dca_fma_eclass_t index); 427 static int dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle, 428 dca_fma_eclass_t eclass_index); 429 430 static void dca_fma_init(dca_t *dca); 431 static void dca_fma_fini(dca_t *dca); 432 static int dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, 433 const void *impl_data); 434 435 436 static dca_device_t dca_devices[] = { 437 /* Broadcom vanilla variants */ 438 { 0x14e4, 0x5820, "Broadcom 5820" }, 439 { 0x14e4, 0x5821, "Broadcom 5821" }, 440 { 0x14e4, 0x5822, "Broadcom 5822" }, 441 { 0x14e4, 0x5825, "Broadcom 5825" }, 442 /* Sun specific OEMd variants */ 443 { 0x108e, 0x5454, "SCA" }, 444 { 0x108e, 0x5455, "SCA 1000" }, 445 { 0x108e, 0x5457, "SCA 500" }, 446 /* subsysid should be 0x5457, but got 0x1 from HW. Assume both here. */ 447 { 0x108e, 0x1, "SCA 500" }, 448 }; 449 450 /* 451 * Device attributes. 452 */ 453 static struct ddi_device_acc_attr dca_regsattr = { 454 DDI_DEVICE_ATTR_V1, 455 DDI_STRUCTURE_LE_ACC, 456 DDI_STRICTORDER_ACC, 457 DDI_FLAGERR_ACC 458 }; 459 460 static struct ddi_device_acc_attr dca_devattr = { 461 DDI_DEVICE_ATTR_V0, 462 DDI_STRUCTURE_LE_ACC, 463 DDI_STRICTORDER_ACC 464 }; 465 466 #if !defined(i386) && !defined(__i386) 467 static struct ddi_device_acc_attr dca_bufattr = { 468 DDI_DEVICE_ATTR_V0, 469 DDI_NEVERSWAP_ACC, 470 DDI_STRICTORDER_ACC 471 }; 472 #endif 473 474 static struct ddi_dma_attr dca_dmaattr = { 475 DMA_ATTR_V0, /* dma_attr_version */ 476 0x0, /* dma_attr_addr_lo */ 477 0xffffffffUL, /* dma_attr_addr_hi */ 478 0x00ffffffUL, /* dma_attr_count_max */ 479 0x40, /* dma_attr_align */ 480 0x40, /* dma_attr_burstsizes */ 481 0x1, /* dma_attr_minxfer */ 482 0x00ffffffUL, /* dma_attr_maxxfer */ 483 0xffffffffUL, /* dma_attr_seg */ 484 #if defined(i386) || defined(__i386) || defined(__amd64) 485 512, /* dma_attr_sgllen */ 486 #else 487 1, /* dma_attr_sgllen */ 488 #endif 489 1, /* dma_attr_granular */ 490 DDI_DMA_FLAGERR /* dma_attr_flags */ 491 }; 492 493 static void *dca_state = NULL; 494 int dca_mindma = 2500; 495 496 /* 497 * FMA eclass string definitions. Note that these string arrays must be 498 * consistent with the dca_fma_eclass_t enum. 499 */ 500 static char *dca_fma_eclass_sca1000[] = { 501 "sca1000.hw.device", 502 "sca1000.hw.timeout", 503 "sca1000.none" 504 }; 505 506 static char *dca_fma_eclass_sca500[] = { 507 "sca500.hw.device", 508 "sca500.hw.timeout", 509 "sca500.none" 510 }; 511 512 /* 513 * DDI entry points. 514 */ 515 int 516 _init(void) 517 { 518 int rv; 519 520 DBG(NULL, DMOD, "dca: in _init"); 521 522 if ((rv = ddi_soft_state_init(&dca_state, sizeof (dca_t), 1)) != 0) { 523 /* this should *never* happen! */ 524 return (rv); 525 } 526 527 if ((rv = mod_install(&modlinkage)) != 0) { 528 /* cleanup here */ 529 ddi_soft_state_fini(&dca_state); 530 return (rv); 531 } 532 533 return (0); 534 } 535 536 int 537 _fini(void) 538 { 539 int rv; 540 541 DBG(NULL, DMOD, "dca: in _fini"); 542 543 if ((rv = mod_remove(&modlinkage)) == 0) { 544 /* cleanup here */ 545 ddi_soft_state_fini(&dca_state); 546 } 547 return (rv); 548 } 549 550 int 551 _info(struct modinfo *modinfop) 552 { 553 DBG(NULL, DMOD, "dca: in _info"); 554 555 return (mod_info(&modlinkage, modinfop)); 556 } 557 558 int 559 dca_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 560 { 561 ddi_acc_handle_t pci; 562 int instance; 563 ddi_iblock_cookie_t ibc; 564 int intr_added = 0; 565 dca_t *dca; 566 ushort_t venid; 567 ushort_t devid; 568 ushort_t revid; 569 ushort_t subsysid; 570 ushort_t subvenid; 571 int i; 572 int ret; 573 char ID[64]; 574 static char *unknowndev = "Unknown device"; 575 576 #if DEBUG 577 /* these are only used for debugging */ 578 ushort_t pcicomm; 579 ushort_t pcistat; 580 uchar_t cachelinesz; 581 uchar_t mingnt; 582 uchar_t maxlat; 583 uchar_t lattmr; 584 #endif 585 586 instance = ddi_get_instance(dip); 587 588 DBG(NULL, DMOD, "dca: in dca_attach() for %d", instance); 589 590 switch (cmd) { 591 case DDI_RESUME: 592 if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) { 593 dca_diperror(dip, "no soft state in detach"); 594 return (DDI_FAILURE); 595 } 596 /* assumption: we won't be DDI_DETACHed until we return */ 597 return (dca_resume(dca)); 598 case DDI_ATTACH: 599 break; 600 default: 601 return (DDI_FAILURE); 602 } 603 604 if (ddi_slaveonly(dip) == DDI_SUCCESS) { 605 dca_diperror(dip, "slot does not support PCI bus-master"); 606 return (DDI_FAILURE); 607 } 608 609 if (ddi_intr_hilevel(dip, 0) != 0) { 610 dca_diperror(dip, "hilevel interrupts not supported"); 611 return (DDI_FAILURE); 612 } 613 614 if (pci_config_setup(dip, &pci) != DDI_SUCCESS) { 615 dca_diperror(dip, "unable to setup PCI config handle"); 616 return (DDI_FAILURE); 617 } 618 619 /* common PCI attributes */ 620 venid = pci_config_get16(pci, PCI_VENID); 621 devid = pci_config_get16(pci, PCI_DEVID); 622 revid = pci_config_get8(pci, PCI_REVID); 623 subvenid = pci_config_get16(pci, PCI_SUBVENID); 624 subsysid = pci_config_get16(pci, PCI_SUBSYSID); 625 626 /* 627 * Broadcom-specific timings. 628 * We disable these timers/counters since they can cause 629 * incorrect false failures when the bus is just a little 630 * bit slow, or busy. 631 */ 632 pci_config_put8(pci, PCI_TRDYTO, 0); 633 pci_config_put8(pci, PCI_RETRIES, 0); 634 635 /* initialize PCI access settings */ 636 pci_config_put16(pci, PCI_COMM, PCICOMM_SEE | 637 PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE); 638 639 /* set up our PCI latency timer */ 640 pci_config_put8(pci, PCI_LATTMR, 0x40); 641 642 #if DEBUG 643 /* read registers (for debugging) */ 644 pcicomm = pci_config_get16(pci, PCI_COMM); 645 pcistat = pci_config_get16(pci, PCI_STATUS); 646 cachelinesz = pci_config_get8(pci, PCI_CACHELINESZ); 647 mingnt = pci_config_get8(pci, PCI_MINGNT); 648 maxlat = pci_config_get8(pci, PCI_MAXLAT); 649 lattmr = pci_config_get8(pci, PCI_LATTMR); 650 #endif 651 652 pci_config_teardown(&pci); 653 654 if (ddi_get_iblock_cookie(dip, 0, &ibc) != DDI_SUCCESS) { 655 dca_diperror(dip, "unable to get iblock cookie"); 656 return (DDI_FAILURE); 657 } 658 659 if (ddi_soft_state_zalloc(dca_state, instance) != DDI_SUCCESS) { 660 dca_diperror(dip, "unable to allocate soft state"); 661 return (DDI_FAILURE); 662 } 663 664 dca = ddi_get_soft_state(dca_state, instance); 665 ASSERT(dca != NULL); 666 dca->dca_dip = dip; 667 WORKLIST(dca, MCR1)->dwl_prov = 0; 668 WORKLIST(dca, MCR2)->dwl_prov = 0; 669 /* figure pagesize */ 670 dca->dca_pagesize = ddi_ptob(dip, 1); 671 672 /* 673 * Search for the device in our supported devices table. This 674 * is here for two reasons. First, we want to ensure that 675 * only Sun-qualified (and presumably Sun-labeled) devices can 676 * be used with this driver. Second, some devices have 677 * specific differences. E.g. the 5821 has support for a 678 * special mode of RC4, deeper queues, power management, and 679 * other changes. Also, the export versions of some of these 680 * chips don't support RC4 or 3DES, so we catch that here. 681 * 682 * Note that we only look at the upper nibble of the device 683 * id, which is used to distinguish export vs. domestic 684 * versions of the chip. (The lower nibble is used for 685 * stepping information.) 686 */ 687 for (i = 0; i < (sizeof (dca_devices) / sizeof (dca_device_t)); i++) { 688 /* 689 * Try to match the subsystem information first. 690 */ 691 if (subvenid && (subvenid == dca_devices[i].dd_vendor_id) && 692 subsysid && (subsysid == dca_devices[i].dd_device_id)) { 693 dca->dca_model = dca_devices[i].dd_model; 694 dca->dca_devid = dca_devices[i].dd_device_id; 695 break; 696 } 697 /* 698 * Failing that, try the generic vendor and device id. 699 * Even if we find a match, we keep searching anyway, 700 * since we would prefer to find a match based on the 701 * subsystem ids. 702 */ 703 if ((venid == dca_devices[i].dd_vendor_id) && 704 (devid == dca_devices[i].dd_device_id)) { 705 dca->dca_model = dca_devices[i].dd_model; 706 dca->dca_devid = dca_devices[i].dd_device_id; 707 } 708 } 709 /* try and handle an unrecognized device */ 710 if (dca->dca_model == NULL) { 711 dca->dca_model = unknowndev; 712 dca_error(dca, "device not recognized, not supported"); 713 DBG(dca, DPCI, "i=%d venid=%x devid=%x rev=%d", 714 i, venid, devid, revid); 715 } 716 717 if (ddi_prop_update_string(DDI_DEV_T_NONE, dip, "description", 718 dca->dca_model) != DDI_SUCCESS) { 719 dca_error(dca, "unable to create description property"); 720 return (DDI_FAILURE); 721 } 722 723 DBG(dca, DPCI, "PCI command=0x%x status=%x cachelinesz=%x", 724 pcicomm, pcistat, cachelinesz); 725 DBG(dca, DPCI, "mingnt=0x%x maxlat=0x%x lattmr=0x%x", 726 mingnt, maxlat, lattmr); 727 728 /* 729 * initialize locks, etc. 730 */ 731 (void) mutex_init(&dca->dca_intrlock, NULL, MUTEX_DRIVER, ibc); 732 733 /* use RNGSHA1 by default */ 734 if (ddi_getprop(DDI_DEV_T_ANY, dip, 735 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "rngdirect", 0) == 0) { 736 dca->dca_flags |= DCA_RNGSHA1; 737 } 738 739 /* initialize FMA */ 740 dca_fma_init(dca); 741 742 /* initialize some key data structures */ 743 if (dca_init(dca) != DDI_SUCCESS) { 744 goto failed; 745 } 746 747 /* initialize kstats */ 748 dca_ksinit(dca); 749 750 /* setup access to registers */ 751 if (ddi_regs_map_setup(dip, 1, (caddr_t *)&dca->dca_regs, 752 0, 0, &dca_regsattr, &dca->dca_regs_handle) != DDI_SUCCESS) { 753 dca_error(dca, "unable to map registers"); 754 goto failed; 755 } 756 757 DBG(dca, DCHATTY, "MCR1 = %x", GETCSR(dca, CSR_MCR1)); 758 DBG(dca, DCHATTY, "CONTROL = %x", GETCSR(dca, CSR_DMACTL)); 759 DBG(dca, DCHATTY, "STATUS = %x", GETCSR(dca, CSR_DMASTAT)); 760 DBG(dca, DCHATTY, "DMAEA = %x", GETCSR(dca, CSR_DMAEA)); 761 DBG(dca, DCHATTY, "MCR2 = %x", GETCSR(dca, CSR_MCR2)); 762 763 /* reset the chip */ 764 if (dca_reset(dca, 0) < 0) { 765 goto failed; 766 } 767 768 /* initialize the chip */ 769 PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64); 770 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 771 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 772 goto failed; 773 } 774 775 /* add the interrupt */ 776 if (ddi_add_intr(dip, 0, &dca->dca_icookie, NULL, dca_intr, 777 (void *)dca) != DDI_SUCCESS) { 778 DBG(dca, DWARN, "ddi_add_intr failed"); 779 goto failed; 780 } else { 781 intr_added = 1; 782 } 783 784 /* enable interrupts on the device */ 785 /* 786 * XXX: Note, 5820A1 errata indicates that this may clobber 787 * bits 24 and 23, which affect the speed of the RNG. Since 788 * we always want to run in full-speed mode, this should be 789 * harmless. 790 */ 791 if (dca->dca_devid == 0x5825) { 792 /* for 5825 - increase the DMA read size */ 793 SETBIT(dca, CSR_DMACTL, 794 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256); 795 } else { 796 SETBIT(dca, CSR_DMACTL, 797 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE); 798 } 799 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 800 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 801 goto failed; 802 } 803 804 /* register MCR1 with the crypto framework */ 805 /* Be careful not to exceed 32 chars */ 806 (void) sprintf(ID, "%s/%d %s", 807 ddi_driver_name(dip), ddi_get_instance(dip), IDENT_SYM); 808 dca_prov_info1.pi_provider_description = ID; 809 dca_prov_info1.pi_provider_dev.pd_hw = dip; 810 dca_prov_info1.pi_provider_handle = dca; 811 if ((ret = crypto_register_provider(&dca_prov_info1, 812 &WORKLIST(dca, MCR1)->dwl_prov)) != CRYPTO_SUCCESS) { 813 cmn_err(CE_WARN, 814 "crypto_register_provider() failed (%d) for MCR1", ret); 815 goto failed; 816 } 817 818 /* register MCR2 with the crypto framework */ 819 /* Be careful not to exceed 32 chars */ 820 (void) sprintf(ID, "%s/%d %s", 821 ddi_driver_name(dip), ddi_get_instance(dip), IDENT_ASYM); 822 dca_prov_info2.pi_provider_description = ID; 823 dca_prov_info2.pi_provider_dev.pd_hw = dip; 824 dca_prov_info2.pi_provider_handle = dca; 825 if ((ret = crypto_register_provider(&dca_prov_info2, 826 &WORKLIST(dca, MCR2)->dwl_prov)) != CRYPTO_SUCCESS) { 827 cmn_err(CE_WARN, 828 "crypto_register_provider() failed (%d) for MCR2", ret); 829 goto failed; 830 } 831 832 crypto_prov_notify(WORKLIST(dca, MCR1)->dwl_prov, 833 CRYPTO_PROVIDER_READY); 834 crypto_prov_notify(WORKLIST(dca, MCR2)->dwl_prov, 835 CRYPTO_PROVIDER_READY); 836 837 /* Initialize the local random number pool for this instance */ 838 if ((ret = dca_random_init(dca)) != CRYPTO_SUCCESS) { 839 goto failed; 840 } 841 842 mutex_enter(&dca->dca_intrlock); 843 dca->dca_jobtid = timeout(dca_jobtimeout, (void *)dca, 844 drv_usectohz(SECOND)); 845 mutex_exit(&dca->dca_intrlock); 846 847 ddi_set_driver_private(dip, (caddr_t)dca); 848 849 ddi_report_dev(dip); 850 851 if (ddi_get_devstate(dca->dca_dip) != DDI_DEVSTATE_UP) { 852 ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_RESTORED); 853 } 854 855 return (DDI_SUCCESS); 856 857 failed: 858 /* unregister from the crypto framework */ 859 if (WORKLIST(dca, MCR1)->dwl_prov != 0) { 860 (void) crypto_unregister_provider( 861 WORKLIST(dca, MCR1)->dwl_prov); 862 } 863 if (WORKLIST(dca, MCR2)->dwl_prov != 0) { 864 (void) crypto_unregister_provider( 865 WORKLIST(dca, MCR2)->dwl_prov); 866 } 867 if (intr_added) { 868 CLRBIT(dca, CSR_DMACTL, 869 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE); 870 /* unregister intr handler */ 871 ddi_remove_intr(dip, 0, dca->dca_icookie); 872 } 873 if (dca->dca_regs_handle) { 874 ddi_regs_map_free(&dca->dca_regs_handle); 875 } 876 if (dca->dca_intrstats) { 877 kstat_delete(dca->dca_intrstats); 878 } 879 if (dca->dca_ksp) { 880 kstat_delete(dca->dca_ksp); 881 } 882 dca_uninit(dca); 883 884 /* finalize FMA */ 885 dca_fma_fini(dca); 886 887 mutex_destroy(&dca->dca_intrlock); 888 ddi_soft_state_free(dca_state, instance); 889 return (DDI_FAILURE); 890 891 } 892 893 int 894 dca_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 895 { 896 int instance; 897 dca_t *dca; 898 timeout_id_t tid; 899 900 instance = ddi_get_instance(dip); 901 902 DBG(NULL, DMOD, "dca: in dca_detach() for %d", instance); 903 904 switch (cmd) { 905 case DDI_SUSPEND: 906 if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) { 907 dca_diperror(dip, "no soft state in detach"); 908 return (DDI_FAILURE); 909 } 910 /* assumption: we won't be DDI_DETACHed until we return */ 911 return (dca_suspend(dca)); 912 913 case DDI_DETACH: 914 break; 915 default: 916 return (DDI_FAILURE); 917 } 918 919 if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) { 920 dca_diperror(dip, "no soft state in detach"); 921 return (DDI_FAILURE); 922 } 923 924 /* 925 * Unregister from kCF. 926 * This needs to be done at the beginning of detach. 927 */ 928 if (WORKLIST(dca, MCR1)->dwl_prov != 0) { 929 if (crypto_unregister_provider( 930 WORKLIST(dca, MCR1)->dwl_prov) != CRYPTO_SUCCESS) { 931 dca_error(dca, "unable to unregister MCR1 from kcf"); 932 return (DDI_FAILURE); 933 } 934 } 935 936 if (WORKLIST(dca, MCR2)->dwl_prov != 0) { 937 if (crypto_unregister_provider( 938 WORKLIST(dca, MCR2)->dwl_prov) != CRYPTO_SUCCESS) { 939 dca_error(dca, "unable to unregister MCR2 from kcf"); 940 return (DDI_FAILURE); 941 } 942 } 943 944 /* 945 * Cleanup the private context list. Once the 946 * crypto_unregister_provider returns, it is safe to do so. 947 */ 948 dca_free_context_list(dca); 949 950 /* Cleanup the local random number pool */ 951 dca_random_fini(dca); 952 953 /* send any jobs in the waitq back to kCF */ 954 dca_rejectjobs(dca); 955 956 /* untimeout the timeouts */ 957 mutex_enter(&dca->dca_intrlock); 958 tid = dca->dca_jobtid; 959 dca->dca_jobtid = 0; 960 mutex_exit(&dca->dca_intrlock); 961 if (tid) { 962 (void) untimeout(tid); 963 } 964 965 /* disable device interrupts */ 966 CLRBIT(dca, CSR_DMACTL, DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE); 967 968 /* unregister interrupt handlers */ 969 ddi_remove_intr(dip, 0, dca->dca_icookie); 970 971 /* release our regs handle */ 972 ddi_regs_map_free(&dca->dca_regs_handle); 973 974 /* toss out kstats */ 975 if (dca->dca_intrstats) { 976 kstat_delete(dca->dca_intrstats); 977 } 978 if (dca->dca_ksp) { 979 kstat_delete(dca->dca_ksp); 980 } 981 982 mutex_destroy(&dca->dca_intrlock); 983 dca_uninit(dca); 984 985 /* finalize FMA */ 986 dca_fma_fini(dca); 987 988 ddi_soft_state_free(dca_state, instance); 989 990 return (DDI_SUCCESS); 991 } 992 993 int 994 dca_resume(dca_t *dca) 995 { 996 ddi_acc_handle_t pci; 997 998 if (pci_config_setup(dca->dca_dip, &pci) != DDI_SUCCESS) { 999 dca_error(dca, "unable to setup PCI config handle"); 1000 return (DDI_FAILURE); 1001 } 1002 1003 /* 1004 * Reprogram registers in PCI configuration space. 1005 */ 1006 1007 /* Broadcom-specific timers -- we disable them. */ 1008 pci_config_put8(pci, PCI_TRDYTO, 0); 1009 pci_config_put8(pci, PCI_RETRIES, 0); 1010 1011 /* initialize PCI access settings */ 1012 pci_config_put16(pci, PCI_COMM, PCICOMM_SEE | 1013 PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE); 1014 1015 /* set up our PCI latency timer */ 1016 pci_config_put8(pci, PCI_LATTMR, 0x40); 1017 1018 pci_config_teardown(&pci); 1019 1020 if (dca_reset(dca, 0) < 0) { 1021 dca_error(dca, "unable to reset device during resume"); 1022 return (DDI_FAILURE); 1023 } 1024 1025 /* 1026 * Now restore the card-specific CSRs. 1027 */ 1028 1029 /* restore endianness settings */ 1030 PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64); 1031 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 1032 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) 1033 return (DDI_FAILURE); 1034 1035 /* restore interrupt enables */ 1036 if (dca->dca_devid == 0x5825) { 1037 /* for 5825 set 256 byte read size to improve performance */ 1038 SETBIT(dca, CSR_DMACTL, 1039 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256); 1040 } else { 1041 SETBIT(dca, CSR_DMACTL, 1042 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE); 1043 } 1044 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 1045 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) 1046 return (DDI_FAILURE); 1047 1048 /* resume scheduling jobs on the device */ 1049 dca_undrain(dca); 1050 1051 return (DDI_SUCCESS); 1052 } 1053 1054 int 1055 dca_suspend(dca_t *dca) 1056 { 1057 if ((dca_drain(dca)) != 0) { 1058 return (DDI_FAILURE); 1059 } 1060 if (dca_reset(dca, 0) < 0) { 1061 dca_error(dca, "unable to reset device during suspend"); 1062 return (DDI_FAILURE); 1063 } 1064 return (DDI_SUCCESS); 1065 } 1066 1067 /* 1068 * Hardware access stuff. 1069 */ 1070 int 1071 dca_reset(dca_t *dca, int failreset) 1072 { 1073 int i; 1074 1075 if (dca->dca_regs_handle == NULL) { 1076 return (-1); 1077 } 1078 1079 PUTCSR(dca, CSR_DMACTL, DMACTL_RESET); 1080 if (!failreset) { 1081 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 1082 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) 1083 return (-1); 1084 } 1085 1086 /* now wait for a reset */ 1087 for (i = 1; i < 100; i++) { 1088 uint32_t dmactl; 1089 drv_usecwait(100); 1090 dmactl = GETCSR(dca, CSR_DMACTL); 1091 if (!failreset) { 1092 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 1093 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) 1094 return (-1); 1095 } 1096 if ((dmactl & DMACTL_RESET) == 0) { 1097 DBG(dca, DCHATTY, "reset in %d usec", i * 100); 1098 return (0); 1099 } 1100 } 1101 if (!failreset) { 1102 dca_failure(dca, DDI_DEVICE_FAULT, 1103 DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR, 1104 "timeout waiting for reset after %d usec", i * 100); 1105 } 1106 return (-1); 1107 } 1108 1109 int 1110 dca_initworklist(dca_t *dca, dca_worklist_t *wlp) 1111 { 1112 int i; 1113 int reqprealloc = wlp->dwl_hiwater + (MAXWORK * MAXREQSPERMCR); 1114 1115 /* 1116 * Set up work queue. 1117 */ 1118 mutex_init(&wlp->dwl_lock, NULL, MUTEX_DRIVER, dca->dca_icookie); 1119 mutex_init(&wlp->dwl_freereqslock, NULL, MUTEX_DRIVER, 1120 dca->dca_icookie); 1121 mutex_init(&wlp->dwl_freelock, NULL, MUTEX_DRIVER, dca->dca_icookie); 1122 cv_init(&wlp->dwl_cv, NULL, CV_DRIVER, NULL); 1123 1124 mutex_enter(&wlp->dwl_lock); 1125 1126 dca_initq(&wlp->dwl_freereqs); 1127 dca_initq(&wlp->dwl_waitq); 1128 dca_initq(&wlp->dwl_freework); 1129 dca_initq(&wlp->dwl_runq); 1130 1131 for (i = 0; i < MAXWORK; i++) { 1132 dca_work_t *workp; 1133 1134 if ((workp = dca_newwork(dca)) == NULL) { 1135 dca_error(dca, "unable to allocate work"); 1136 mutex_exit(&wlp->dwl_lock); 1137 return (DDI_FAILURE); 1138 } 1139 workp->dw_wlp = wlp; 1140 dca_freework(workp); 1141 } 1142 mutex_exit(&wlp->dwl_lock); 1143 1144 for (i = 0; i < reqprealloc; i++) { 1145 dca_request_t *reqp; 1146 1147 if ((reqp = dca_newreq(dca)) == NULL) { 1148 dca_error(dca, "unable to allocate request"); 1149 return (DDI_FAILURE); 1150 } 1151 reqp->dr_dca = dca; 1152 reqp->dr_wlp = wlp; 1153 dca_freereq(reqp); 1154 } 1155 return (DDI_SUCCESS); 1156 } 1157 1158 int 1159 dca_init(dca_t *dca) 1160 { 1161 dca_worklist_t *wlp; 1162 1163 /* Initialize the private context list and the corresponding lock. */ 1164 mutex_init(&dca->dca_ctx_list_lock, NULL, MUTEX_DRIVER, NULL); 1165 dca_initq(&dca->dca_ctx_list); 1166 1167 /* 1168 * MCR1 algorithms. 1169 */ 1170 wlp = WORKLIST(dca, MCR1); 1171 (void) sprintf(wlp->dwl_name, "dca%d:mcr1", 1172 ddi_get_instance(dca->dca_dip)); 1173 wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY, 1174 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, 1175 "mcr1_lowater", MCR1LOWATER); 1176 wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY, 1177 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, 1178 "mcr1_hiwater", MCR1HIWATER); 1179 wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY, 1180 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, 1181 "mcr1_maxreqs", MCR1MAXREQS), MAXREQSPERMCR); 1182 wlp->dwl_dca = dca; 1183 wlp->dwl_mcr = MCR1; 1184 if (dca_initworklist(dca, wlp) != DDI_SUCCESS) { 1185 return (DDI_FAILURE); 1186 } 1187 1188 /* 1189 * MCR2 algorithms. 1190 */ 1191 wlp = WORKLIST(dca, MCR2); 1192 (void) sprintf(wlp->dwl_name, "dca%d:mcr2", 1193 ddi_get_instance(dca->dca_dip)); 1194 wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY, 1195 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, 1196 "mcr2_lowater", MCR2LOWATER); 1197 wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY, 1198 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, 1199 "mcr2_hiwater", MCR2HIWATER); 1200 wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY, 1201 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, 1202 "mcr2_maxreqs", MCR2MAXREQS), MAXREQSPERMCR); 1203 wlp->dwl_dca = dca; 1204 wlp->dwl_mcr = MCR2; 1205 if (dca_initworklist(dca, wlp) != DDI_SUCCESS) { 1206 return (DDI_FAILURE); 1207 } 1208 return (DDI_SUCCESS); 1209 } 1210 1211 /* 1212 * Uninitialize worklists. This routine should only be called when no 1213 * active jobs (hence DMA mappings) exist. One way to ensure this is 1214 * to unregister from kCF before calling this routine. (This is done 1215 * e.g. in detach(9e).) 1216 */ 1217 void 1218 dca_uninit(dca_t *dca) 1219 { 1220 int mcr; 1221 1222 mutex_destroy(&dca->dca_ctx_list_lock); 1223 1224 for (mcr = MCR1; mcr <= MCR2; mcr++) { 1225 dca_worklist_t *wlp = WORKLIST(dca, mcr); 1226 dca_work_t *workp; 1227 dca_request_t *reqp; 1228 1229 if (dca->dca_regs_handle == NULL) { 1230 continue; 1231 } 1232 1233 mutex_enter(&wlp->dwl_lock); 1234 while ((workp = dca_getwork(dca, mcr)) != NULL) { 1235 dca_destroywork(workp); 1236 } 1237 mutex_exit(&wlp->dwl_lock); 1238 while ((reqp = dca_getreq(dca, mcr, 0)) != NULL) { 1239 dca_destroyreq(reqp); 1240 } 1241 1242 mutex_destroy(&wlp->dwl_lock); 1243 mutex_destroy(&wlp->dwl_freereqslock); 1244 mutex_destroy(&wlp->dwl_freelock); 1245 cv_destroy(&wlp->dwl_cv); 1246 wlp->dwl_prov = 0; 1247 } 1248 } 1249 1250 static void 1251 dca_enlist2(dca_listnode_t *q, dca_listnode_t *node, kmutex_t *lock) 1252 { 1253 if (!q || !node) 1254 return; 1255 1256 mutex_enter(lock); 1257 node->dl_next2 = q; 1258 node->dl_prev2 = q->dl_prev2; 1259 node->dl_next2->dl_prev2 = node; 1260 node->dl_prev2->dl_next2 = node; 1261 mutex_exit(lock); 1262 } 1263 1264 static void 1265 dca_rmlist2(dca_listnode_t *node, kmutex_t *lock) 1266 { 1267 if (!node) 1268 return; 1269 1270 mutex_enter(lock); 1271 node->dl_next2->dl_prev2 = node->dl_prev2; 1272 node->dl_prev2->dl_next2 = node->dl_next2; 1273 node->dl_next2 = NULL; 1274 node->dl_prev2 = NULL; 1275 mutex_exit(lock); 1276 } 1277 1278 static dca_listnode_t * 1279 dca_delist2(dca_listnode_t *q, kmutex_t *lock) 1280 { 1281 dca_listnode_t *node; 1282 1283 mutex_enter(lock); 1284 if ((node = q->dl_next2) == q) { 1285 mutex_exit(lock); 1286 return (NULL); 1287 } 1288 1289 node->dl_next2->dl_prev2 = node->dl_prev2; 1290 node->dl_prev2->dl_next2 = node->dl_next2; 1291 node->dl_next2 = NULL; 1292 node->dl_prev2 = NULL; 1293 mutex_exit(lock); 1294 1295 return (node); 1296 } 1297 1298 void 1299 dca_initq(dca_listnode_t *q) 1300 { 1301 q->dl_next = q; 1302 q->dl_prev = q; 1303 q->dl_next2 = q; 1304 q->dl_prev2 = q; 1305 } 1306 1307 void 1308 dca_enqueue(dca_listnode_t *q, dca_listnode_t *node) 1309 { 1310 /* 1311 * Enqueue submits at the "tail" of the list, i.e. just 1312 * behind the sentinel. 1313 */ 1314 node->dl_next = q; 1315 node->dl_prev = q->dl_prev; 1316 node->dl_next->dl_prev = node; 1317 node->dl_prev->dl_next = node; 1318 } 1319 1320 void 1321 dca_rmqueue(dca_listnode_t *node) 1322 { 1323 node->dl_next->dl_prev = node->dl_prev; 1324 node->dl_prev->dl_next = node->dl_next; 1325 node->dl_next = NULL; 1326 node->dl_prev = NULL; 1327 } 1328 1329 dca_listnode_t * 1330 dca_dequeue(dca_listnode_t *q) 1331 { 1332 dca_listnode_t *node; 1333 /* 1334 * Dequeue takes from the "head" of the list, i.e. just after 1335 * the sentinel. 1336 */ 1337 if ((node = q->dl_next) == q) { 1338 /* queue is empty */ 1339 return (NULL); 1340 } 1341 dca_rmqueue(node); 1342 return (node); 1343 } 1344 1345 /* this is the opposite of dequeue, it takes things off in LIFO order */ 1346 dca_listnode_t * 1347 dca_unqueue(dca_listnode_t *q) 1348 { 1349 dca_listnode_t *node; 1350 /* 1351 * unqueue takes from the "tail" of the list, i.e. just before 1352 * the sentinel. 1353 */ 1354 if ((node = q->dl_prev) == q) { 1355 /* queue is empty */ 1356 return (NULL); 1357 } 1358 dca_rmqueue(node); 1359 return (node); 1360 } 1361 1362 dca_listnode_t * 1363 dca_peekqueue(dca_listnode_t *q) 1364 { 1365 dca_listnode_t *node; 1366 1367 if ((node = q->dl_next) == q) { 1368 return (NULL); 1369 } else { 1370 return (node); 1371 } 1372 } 1373 1374 /* 1375 * Interrupt service routine. 1376 */ 1377 uint_t 1378 dca_intr(char *arg) 1379 { 1380 dca_t *dca = (dca_t *)arg; 1381 uint32_t status; 1382 1383 mutex_enter(&dca->dca_intrlock); 1384 status = GETCSR(dca, CSR_DMASTAT); 1385 PUTCSR(dca, CSR_DMASTAT, status & DMASTAT_INTERRUPTS); 1386 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 1387 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 1388 mutex_exit(&dca->dca_intrlock); 1389 return ((uint_t)DDI_FAILURE); 1390 } 1391 1392 DBG(dca, DINTR, "interrupted, status = 0x%x!", status); 1393 1394 if ((status & DMASTAT_INTERRUPTS) == 0) { 1395 /* increment spurious interrupt kstat */ 1396 if (dca->dca_intrstats) { 1397 KIOIP(dca)->intrs[KSTAT_INTR_SPURIOUS]++; 1398 } 1399 mutex_exit(&dca->dca_intrlock); 1400 return (DDI_INTR_UNCLAIMED); 1401 } 1402 1403 if (dca->dca_intrstats) { 1404 KIOIP(dca)->intrs[KSTAT_INTR_HARD]++; 1405 } 1406 if (status & DMASTAT_MCR1INT) { 1407 DBG(dca, DINTR, "MCR1 interrupted"); 1408 mutex_enter(&(WORKLIST(dca, MCR1)->dwl_lock)); 1409 dca_schedule(dca, MCR1); 1410 dca_reclaim(dca, MCR1); 1411 mutex_exit(&(WORKLIST(dca, MCR1)->dwl_lock)); 1412 } 1413 1414 if (status & DMASTAT_MCR2INT) { 1415 DBG(dca, DINTR, "MCR2 interrupted"); 1416 mutex_enter(&(WORKLIST(dca, MCR2)->dwl_lock)); 1417 dca_schedule(dca, MCR2); 1418 dca_reclaim(dca, MCR2); 1419 mutex_exit(&(WORKLIST(dca, MCR2)->dwl_lock)); 1420 } 1421 1422 if (status & DMASTAT_ERRINT) { 1423 uint32_t erraddr; 1424 erraddr = GETCSR(dca, CSR_DMAEA); 1425 mutex_exit(&dca->dca_intrlock); 1426 1427 /* 1428 * bit 1 of the error address indicates failure during 1429 * read if set, during write otherwise. 1430 */ 1431 dca_failure(dca, DDI_DEVICE_FAULT, 1432 DCA_FM_ECLASS_HW_DEVICE, dca_ena(0), CRYPTO_DEVICE_ERROR, 1433 "DMA master access error %s address 0x%x", 1434 erraddr & 0x1 ? "reading" : "writing", erraddr & ~1); 1435 return (DDI_INTR_CLAIMED); 1436 } 1437 1438 mutex_exit(&dca->dca_intrlock); 1439 1440 return (DDI_INTR_CLAIMED); 1441 } 1442 1443 /* 1444 * Reverse a string of bytes from s1 into s2. The reversal happens 1445 * from the tail of s1. If len1 < len2, then null bytes will be 1446 * padded to the end of s2. If len2 < len1, then (presumably null) 1447 * bytes will be dropped from the start of s1. 1448 * 1449 * The rationale here is that when s1 (source) is shorter, then we 1450 * are reversing from big-endian ordering, into device ordering, and 1451 * want to add some extra nulls to the tail (MSB) side of the device. 1452 * 1453 * Similarly, when s2 (dest) is shorter, then we are truncating what 1454 * are presumably null MSB bits from the device. 1455 * 1456 * There is an expectation when reversing from the device back into 1457 * big-endian, that the number of bytes to reverse and the target size 1458 * will match, and no truncation or padding occurs. 1459 */ 1460 void 1461 dca_reverse(void *s1, void *s2, int len1, int len2) 1462 { 1463 caddr_t src, dst; 1464 1465 if (len1 == 0) { 1466 if (len2) { 1467 bzero(s2, len2); 1468 } 1469 return; 1470 } 1471 src = (caddr_t)s1 + len1 - 1; 1472 dst = s2; 1473 while ((src >= (caddr_t)s1) && (len2)) { 1474 *dst++ = *src--; 1475 len2--; 1476 } 1477 while (len2 > 0) { 1478 *dst++ = 0; 1479 len2--; 1480 } 1481 } 1482 1483 uint16_t 1484 dca_padfull(int num) 1485 { 1486 if (num <= 512) { 1487 return (BITS2BYTES(512)); 1488 } 1489 if (num <= 768) { 1490 return (BITS2BYTES(768)); 1491 } 1492 if (num <= 1024) { 1493 return (BITS2BYTES(1024)); 1494 } 1495 if (num <= 1536) { 1496 return (BITS2BYTES(1536)); 1497 } 1498 if (num <= 2048) { 1499 return (BITS2BYTES(2048)); 1500 } 1501 return (0); 1502 } 1503 1504 uint16_t 1505 dca_padhalf(int num) 1506 { 1507 if (num <= 256) { 1508 return (BITS2BYTES(256)); 1509 } 1510 if (num <= 384) { 1511 return (BITS2BYTES(384)); 1512 } 1513 if (num <= 512) { 1514 return (BITS2BYTES(512)); 1515 } 1516 if (num <= 768) { 1517 return (BITS2BYTES(768)); 1518 } 1519 if (num <= 1024) { 1520 return (BITS2BYTES(1024)); 1521 } 1522 return (0); 1523 } 1524 1525 dca_work_t * 1526 dca_newwork(dca_t *dca) 1527 { 1528 dca_work_t *workp; 1529 size_t size; 1530 ddi_dma_cookie_t c; 1531 unsigned nc; 1532 int rv; 1533 1534 workp = kmem_zalloc(sizeof (dca_work_t), KM_SLEEP); 1535 1536 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, 1537 DDI_DMA_SLEEP, NULL, &workp->dw_mcr_dmah); 1538 if (rv != 0) { 1539 dca_error(dca, "unable to alloc MCR DMA handle"); 1540 dca_destroywork(workp); 1541 return (NULL); 1542 } 1543 1544 rv = ddi_dma_mem_alloc(workp->dw_mcr_dmah, 1545 ROUNDUP(MCR_SIZE, dca->dca_pagesize), 1546 &dca_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, 1547 &workp->dw_mcr_kaddr, &size, &workp->dw_mcr_acch); 1548 if (rv != 0) { 1549 dca_error(dca, "unable to alloc MCR DMA memory"); 1550 dca_destroywork(workp); 1551 return (NULL); 1552 } 1553 1554 rv = ddi_dma_addr_bind_handle(workp->dw_mcr_dmah, NULL, 1555 workp->dw_mcr_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_RDWR, 1556 DDI_DMA_SLEEP, NULL, &c, &nc); 1557 if (rv != DDI_DMA_MAPPED) { 1558 dca_error(dca, "unable to map MCR DMA memory"); 1559 dca_destroywork(workp); 1560 return (NULL); 1561 } 1562 1563 workp->dw_mcr_paddr = c.dmac_address; 1564 return (workp); 1565 } 1566 1567 void 1568 dca_destroywork(dca_work_t *workp) 1569 { 1570 if (workp->dw_mcr_paddr) { 1571 (void) ddi_dma_unbind_handle(workp->dw_mcr_dmah); 1572 } 1573 if (workp->dw_mcr_acch) { 1574 ddi_dma_mem_free(&workp->dw_mcr_acch); 1575 } 1576 if (workp->dw_mcr_dmah) { 1577 ddi_dma_free_handle(&workp->dw_mcr_dmah); 1578 } 1579 kmem_free(workp, sizeof (dca_work_t)); 1580 } 1581 1582 dca_request_t * 1583 dca_newreq(dca_t *dca) 1584 { 1585 dca_request_t *reqp; 1586 size_t size; 1587 ddi_dma_cookie_t c; 1588 unsigned nc; 1589 int rv; 1590 int n_chain = 0; 1591 1592 size = (DESC_SIZE * MAXFRAGS) + CTX_MAXLENGTH; 1593 1594 reqp = kmem_zalloc(sizeof (dca_request_t), KM_SLEEP); 1595 1596 reqp->dr_dca = dca; 1597 1598 /* 1599 * Setup the DMA region for the context and descriptors. 1600 */ 1601 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, DDI_DMA_SLEEP, 1602 NULL, &reqp->dr_ctx_dmah); 1603 if (rv != DDI_SUCCESS) { 1604 dca_error(dca, "failure allocating request DMA handle"); 1605 dca_destroyreq(reqp); 1606 return (NULL); 1607 } 1608 1609 /* for driver hardening, allocate in whole pages */ 1610 rv = ddi_dma_mem_alloc(reqp->dr_ctx_dmah, 1611 ROUNDUP(size, dca->dca_pagesize), &dca_devattr, DDI_DMA_CONSISTENT, 1612 DDI_DMA_SLEEP, NULL, &reqp->dr_ctx_kaddr, &size, 1613 &reqp->dr_ctx_acch); 1614 if (rv != DDI_SUCCESS) { 1615 dca_error(dca, "unable to alloc request DMA memory"); 1616 dca_destroyreq(reqp); 1617 return (NULL); 1618 } 1619 1620 rv = ddi_dma_addr_bind_handle(reqp->dr_ctx_dmah, NULL, 1621 reqp->dr_ctx_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_WRITE, 1622 DDI_DMA_SLEEP, 0, &c, &nc); 1623 if (rv != DDI_DMA_MAPPED) { 1624 dca_error(dca, "failed binding request DMA handle"); 1625 dca_destroyreq(reqp); 1626 return (NULL); 1627 } 1628 reqp->dr_ctx_paddr = c.dmac_address; 1629 1630 reqp->dr_dma_size = size; 1631 1632 /* 1633 * Set up the dma for our scratch/shared buffers. 1634 */ 1635 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, 1636 DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_dmah); 1637 if (rv != DDI_SUCCESS) { 1638 dca_error(dca, "failure allocating ibuf DMA handle"); 1639 dca_destroyreq(reqp); 1640 return (NULL); 1641 } 1642 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, 1643 DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_dmah); 1644 if (rv != DDI_SUCCESS) { 1645 dca_error(dca, "failure allocating obuf DMA handle"); 1646 dca_destroyreq(reqp); 1647 return (NULL); 1648 } 1649 1650 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, 1651 DDI_DMA_SLEEP, NULL, &reqp->dr_chain_in_dmah); 1652 if (rv != DDI_SUCCESS) { 1653 dca_error(dca, "failure allocating chain_in DMA handle"); 1654 dca_destroyreq(reqp); 1655 return (NULL); 1656 } 1657 1658 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, 1659 DDI_DMA_SLEEP, NULL, &reqp->dr_chain_out_dmah); 1660 if (rv != DDI_SUCCESS) { 1661 dca_error(dca, "failure allocating chain_out DMA handle"); 1662 dca_destroyreq(reqp); 1663 return (NULL); 1664 } 1665 1666 /* 1667 * for driver hardening, allocate in whole pages. 1668 */ 1669 size = ROUNDUP(MAXPACKET, dca->dca_pagesize); 1670 #if defined(i386) || defined(__i386) 1671 /* 1672 * Use kmem_alloc instead of ddi_dma_mem_alloc here since the latter 1673 * may fail on x86 platform if a physically contiguous memory chunk 1674 * cannot be found. From initial testing, we did not see performance 1675 * degradation as seen on Sparc. 1676 */ 1677 if ((reqp->dr_ibuf_kaddr = kmem_alloc(size, KM_SLEEP)) == NULL) { 1678 dca_error(dca, "unable to alloc request ibuf memory"); 1679 dca_destroyreq(reqp); 1680 return (NULL); 1681 } 1682 if ((reqp->dr_obuf_kaddr = kmem_alloc(size, KM_SLEEP)) == NULL) { 1683 dca_error(dca, "unable to alloc request obuf memory"); 1684 dca_destroyreq(reqp); 1685 return (NULL); 1686 } 1687 #else 1688 /* 1689 * We could kmem_alloc for Sparc too. However, it gives worse 1690 * performance when transferring more than one page data. For example, 1691 * using 4 threads and 12032 byte data and 3DES on 900MHZ Sparc system, 1692 * kmem_alloc uses 80% CPU and ddi_dma_mem_alloc uses 50% CPU for 1693 * the same throughput. 1694 */ 1695 rv = ddi_dma_mem_alloc(reqp->dr_ibuf_dmah, 1696 size, &dca_bufattr, 1697 DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_kaddr, 1698 &size, &reqp->dr_ibuf_acch); 1699 if (rv != DDI_SUCCESS) { 1700 dca_error(dca, "unable to alloc request DMA memory"); 1701 dca_destroyreq(reqp); 1702 return (NULL); 1703 } 1704 1705 rv = ddi_dma_mem_alloc(reqp->dr_obuf_dmah, 1706 size, &dca_bufattr, 1707 DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_kaddr, 1708 &size, &reqp->dr_obuf_acch); 1709 if (rv != DDI_SUCCESS) { 1710 dca_error(dca, "unable to alloc request DMA memory"); 1711 dca_destroyreq(reqp); 1712 return (NULL); 1713 } 1714 #endif 1715 1716 /* Skip the used portion in the context page */ 1717 reqp->dr_offset = CTX_MAXLENGTH; 1718 if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset, 1719 reqp->dr_ibuf_kaddr, reqp->dr_ibuf_dmah, 1720 DDI_DMA_WRITE | DDI_DMA_STREAMING, 1721 &reqp->dr_ibuf_head, &n_chain)) != DDI_SUCCESS) { 1722 (void) dca_destroyreq(reqp); 1723 return (NULL); 1724 } 1725 reqp->dr_ibuf_paddr = reqp->dr_ibuf_head.dc_buffer_paddr; 1726 /* Skip the space used by the input buffer */ 1727 reqp->dr_offset += DESC_SIZE * n_chain; 1728 1729 if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset, 1730 reqp->dr_obuf_kaddr, reqp->dr_obuf_dmah, 1731 DDI_DMA_READ | DDI_DMA_STREAMING, 1732 &reqp->dr_obuf_head, &n_chain)) != DDI_SUCCESS) { 1733 (void) dca_destroyreq(reqp); 1734 return (NULL); 1735 } 1736 reqp->dr_obuf_paddr = reqp->dr_obuf_head.dc_buffer_paddr; 1737 /* Skip the space used by the output buffer */ 1738 reqp->dr_offset += DESC_SIZE * n_chain; 1739 1740 DBG(dca, DCHATTY, "CTX is 0x%p, phys 0x%x, len %d", 1741 reqp->dr_ctx_kaddr, reqp->dr_ctx_paddr, CTX_MAXLENGTH); 1742 return (reqp); 1743 } 1744 1745 void 1746 dca_destroyreq(dca_request_t *reqp) 1747 { 1748 #if defined(i386) || defined(__i386) 1749 dca_t *dca = reqp->dr_dca; 1750 size_t size = ROUNDUP(MAXPACKET, dca->dca_pagesize); 1751 #endif 1752 1753 /* 1754 * Clean up DMA for the context structure. 1755 */ 1756 if (reqp->dr_ctx_paddr) { 1757 (void) ddi_dma_unbind_handle(reqp->dr_ctx_dmah); 1758 } 1759 1760 if (reqp->dr_ctx_acch) { 1761 ddi_dma_mem_free(&reqp->dr_ctx_acch); 1762 } 1763 1764 if (reqp->dr_ctx_dmah) { 1765 ddi_dma_free_handle(&reqp->dr_ctx_dmah); 1766 } 1767 1768 /* 1769 * Clean up DMA for the scratch buffer. 1770 */ 1771 #if defined(i386) || defined(__i386) 1772 if (reqp->dr_ibuf_dmah) { 1773 (void) ddi_dma_unbind_handle(reqp->dr_ibuf_dmah); 1774 ddi_dma_free_handle(&reqp->dr_ibuf_dmah); 1775 } 1776 if (reqp->dr_obuf_dmah) { 1777 (void) ddi_dma_unbind_handle(reqp->dr_obuf_dmah); 1778 ddi_dma_free_handle(&reqp->dr_obuf_dmah); 1779 } 1780 1781 kmem_free(reqp->dr_ibuf_kaddr, size); 1782 kmem_free(reqp->dr_obuf_kaddr, size); 1783 #else 1784 if (reqp->dr_ibuf_paddr) { 1785 (void) ddi_dma_unbind_handle(reqp->dr_ibuf_dmah); 1786 } 1787 if (reqp->dr_obuf_paddr) { 1788 (void) ddi_dma_unbind_handle(reqp->dr_obuf_dmah); 1789 } 1790 1791 if (reqp->dr_ibuf_acch) { 1792 ddi_dma_mem_free(&reqp->dr_ibuf_acch); 1793 } 1794 if (reqp->dr_obuf_acch) { 1795 ddi_dma_mem_free(&reqp->dr_obuf_acch); 1796 } 1797 1798 if (reqp->dr_ibuf_dmah) { 1799 ddi_dma_free_handle(&reqp->dr_ibuf_dmah); 1800 } 1801 if (reqp->dr_obuf_dmah) { 1802 ddi_dma_free_handle(&reqp->dr_obuf_dmah); 1803 } 1804 #endif 1805 /* 1806 * These two DMA handles should have been unbinded in 1807 * dca_unbindchains() function 1808 */ 1809 if (reqp->dr_chain_in_dmah) { 1810 ddi_dma_free_handle(&reqp->dr_chain_in_dmah); 1811 } 1812 if (reqp->dr_chain_out_dmah) { 1813 ddi_dma_free_handle(&reqp->dr_chain_out_dmah); 1814 } 1815 1816 kmem_free(reqp, sizeof (dca_request_t)); 1817 } 1818 1819 dca_work_t * 1820 dca_getwork(dca_t *dca, int mcr) 1821 { 1822 dca_worklist_t *wlp = WORKLIST(dca, mcr); 1823 dca_work_t *workp; 1824 1825 mutex_enter(&wlp->dwl_freelock); 1826 workp = (dca_work_t *)dca_dequeue(&wlp->dwl_freework); 1827 mutex_exit(&wlp->dwl_freelock); 1828 if (workp) { 1829 int nreqs; 1830 bzero(workp->dw_mcr_kaddr, 8); 1831 1832 /* clear out old requests */ 1833 for (nreqs = 0; nreqs < MAXREQSPERMCR; nreqs++) { 1834 workp->dw_reqs[nreqs] = NULL; 1835 } 1836 } 1837 return (workp); 1838 } 1839 1840 void 1841 dca_freework(dca_work_t *workp) 1842 { 1843 mutex_enter(&workp->dw_wlp->dwl_freelock); 1844 dca_enqueue(&workp->dw_wlp->dwl_freework, (dca_listnode_t *)workp); 1845 mutex_exit(&workp->dw_wlp->dwl_freelock); 1846 } 1847 1848 dca_request_t * 1849 dca_getreq(dca_t *dca, int mcr, int tryhard) 1850 { 1851 dca_worklist_t *wlp = WORKLIST(dca, mcr); 1852 dca_request_t *reqp; 1853 1854 mutex_enter(&wlp->dwl_freereqslock); 1855 reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_freereqs); 1856 mutex_exit(&wlp->dwl_freereqslock); 1857 if (reqp) { 1858 reqp->dr_flags = 0; 1859 reqp->dr_callback = NULL; 1860 } else if (tryhard) { 1861 /* 1862 * failed to get a free one, try an allocation, the hard way. 1863 * XXX: Kstat desired here. 1864 */ 1865 if ((reqp = dca_newreq(dca)) != NULL) { 1866 reqp->dr_wlp = wlp; 1867 reqp->dr_dca = dca; 1868 reqp->dr_flags = 0; 1869 reqp->dr_callback = NULL; 1870 } 1871 } 1872 return (reqp); 1873 } 1874 1875 void 1876 dca_freereq(dca_request_t *reqp) 1877 { 1878 reqp->dr_kcf_req = NULL; 1879 if (!(reqp->dr_flags & DR_NOCACHE)) { 1880 mutex_enter(&reqp->dr_wlp->dwl_freereqslock); 1881 dca_enqueue(&reqp->dr_wlp->dwl_freereqs, 1882 (dca_listnode_t *)reqp); 1883 mutex_exit(&reqp->dr_wlp->dwl_freereqslock); 1884 } 1885 } 1886 1887 /* 1888 * Binds user buffers to DMA handles dynamically. On Sparc, a user buffer 1889 * is mapped to a single physical address. On x86, a user buffer is mapped 1890 * to multiple physical addresses. These physical addresses are chained 1891 * using the method specified in Broadcom BCM5820 specification. 1892 */ 1893 int 1894 dca_bindchains(dca_request_t *reqp, size_t incnt, size_t outcnt) 1895 { 1896 int rv; 1897 caddr_t kaddr; 1898 uint_t flags; 1899 int n_chain = 0; 1900 1901 if (reqp->dr_flags & DR_INPLACE) { 1902 flags = DDI_DMA_RDWR | DDI_DMA_CONSISTENT; 1903 } else { 1904 flags = DDI_DMA_WRITE | DDI_DMA_STREAMING; 1905 } 1906 1907 /* first the input */ 1908 if (incnt) { 1909 if ((kaddr = dca_bufdaddr(reqp->dr_in)) == NULL) { 1910 DBG(NULL, DWARN, "unrecognised crypto data format"); 1911 return (DDI_FAILURE); 1912 } 1913 if ((rv = dca_bindchains_one(reqp, incnt, reqp->dr_offset, 1914 kaddr, reqp->dr_chain_in_dmah, flags, 1915 &reqp->dr_chain_in_head, &n_chain)) != DDI_SUCCESS) { 1916 (void) dca_unbindchains(reqp); 1917 return (rv); 1918 } 1919 1920 /* 1921 * The offset and length are altered by the calling routine 1922 * reqp->dr_in->cd_offset += incnt; 1923 * reqp->dr_in->cd_length -= incnt; 1924 */ 1925 /* Save the first one in the chain for MCR */ 1926 reqp->dr_in_paddr = reqp->dr_chain_in_head.dc_buffer_paddr; 1927 reqp->dr_in_next = reqp->dr_chain_in_head.dc_next_paddr; 1928 reqp->dr_in_len = reqp->dr_chain_in_head.dc_buffer_length; 1929 } else { 1930 reqp->dr_in_paddr = 0; 1931 reqp->dr_in_next = 0; 1932 reqp->dr_in_len = 0; 1933 } 1934 1935 if (reqp->dr_flags & DR_INPLACE) { 1936 reqp->dr_out_paddr = reqp->dr_in_paddr; 1937 reqp->dr_out_len = reqp->dr_in_len; 1938 reqp->dr_out_next = reqp->dr_in_next; 1939 return (DDI_SUCCESS); 1940 } 1941 1942 /* then the output */ 1943 if (outcnt) { 1944 flags = DDI_DMA_READ | DDI_DMA_STREAMING; 1945 if ((kaddr = dca_bufdaddr_out(reqp->dr_out)) == NULL) { 1946 DBG(NULL, DWARN, "unrecognised crypto data format"); 1947 (void) dca_unbindchains(reqp); 1948 return (DDI_FAILURE); 1949 } 1950 rv = dca_bindchains_one(reqp, outcnt, reqp->dr_offset + 1951 n_chain * DESC_SIZE, kaddr, reqp->dr_chain_out_dmah, 1952 flags, &reqp->dr_chain_out_head, &n_chain); 1953 if (rv != DDI_SUCCESS) { 1954 (void) dca_unbindchains(reqp); 1955 return (DDI_FAILURE); 1956 } 1957 1958 /* Save the first one in the chain for MCR */ 1959 reqp->dr_out_paddr = reqp->dr_chain_out_head.dc_buffer_paddr; 1960 reqp->dr_out_next = reqp->dr_chain_out_head.dc_next_paddr; 1961 reqp->dr_out_len = reqp->dr_chain_out_head.dc_buffer_length; 1962 } else { 1963 reqp->dr_out_paddr = 0; 1964 reqp->dr_out_next = 0; 1965 reqp->dr_out_len = 0; 1966 } 1967 1968 return (DDI_SUCCESS); 1969 } 1970 1971 /* 1972 * Unbind the user buffers from the DMA handles. 1973 */ 1974 int 1975 dca_unbindchains(dca_request_t *reqp) 1976 { 1977 int rv = DDI_SUCCESS; 1978 int rv1 = DDI_SUCCESS; 1979 1980 /* Clear the input chain */ 1981 if (reqp->dr_chain_in_head.dc_buffer_paddr != 0) { 1982 (void) ddi_dma_unbind_handle(reqp->dr_chain_in_dmah); 1983 reqp->dr_chain_in_head.dc_buffer_paddr = 0; 1984 } 1985 1986 if (reqp->dr_flags & DR_INPLACE) { 1987 return (rv); 1988 } 1989 1990 /* Clear the output chain */ 1991 if (reqp->dr_chain_out_head.dc_buffer_paddr != 0) { 1992 (void) ddi_dma_unbind_handle(reqp->dr_chain_out_dmah); 1993 reqp->dr_chain_out_head.dc_buffer_paddr = 0; 1994 } 1995 1996 return ((rv != DDI_SUCCESS)? rv : rv1); 1997 } 1998 1999 /* 2000 * Build either input chain or output chain. It is single-item chain for Sparc, 2001 * and possible mutiple-item chain for x86. 2002 */ 2003 static int 2004 dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset, 2005 caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags, 2006 dca_chain_t *head, int *n_chain) 2007 { 2008 ddi_dma_cookie_t c; 2009 uint_t nc; 2010 int rv; 2011 caddr_t chain_kaddr_pre; 2012 caddr_t chain_kaddr; 2013 uint32_t chain_paddr; 2014 int i; 2015 2016 /* Advance past the context structure to the starting address */ 2017 chain_paddr = reqp->dr_ctx_paddr + dr_offset; 2018 chain_kaddr = reqp->dr_ctx_kaddr + dr_offset; 2019 2020 /* 2021 * Bind the kernel address to the DMA handle. On x86, the actual 2022 * buffer is mapped into multiple physical addresses. On Sparc, 2023 * the actual buffer is mapped into a single address. 2024 */ 2025 rv = ddi_dma_addr_bind_handle(handle, 2026 NULL, kaddr, cnt, flags, DDI_DMA_DONTWAIT, NULL, &c, &nc); 2027 if (rv != DDI_DMA_MAPPED) { 2028 return (DDI_FAILURE); 2029 } 2030 2031 (void) ddi_dma_sync(handle, 0, cnt, DDI_DMA_SYNC_FORDEV); 2032 if ((rv = dca_check_dma_handle(reqp->dr_dca, handle, 2033 DCA_FM_ECLASS_NONE)) != DDI_SUCCESS) { 2034 reqp->destroy = TRUE; 2035 return (rv); 2036 } 2037 2038 *n_chain = nc; 2039 2040 /* Setup the data buffer chain for DMA transfer */ 2041 chain_kaddr_pre = NULL; 2042 head->dc_buffer_paddr = 0; 2043 head->dc_next_paddr = 0; 2044 head->dc_buffer_length = 0; 2045 for (i = 0; i < nc; i++) { 2046 /* PIO */ 2047 PUTDESC32(reqp, chain_kaddr, DESC_BUFADDR, c.dmac_address); 2048 PUTDESC16(reqp, chain_kaddr, DESC_RSVD, 0); 2049 PUTDESC16(reqp, chain_kaddr, DESC_LENGTH, c.dmac_size); 2050 2051 /* Remember the head of the chain */ 2052 if (head->dc_buffer_paddr == 0) { 2053 head->dc_buffer_paddr = c.dmac_address; 2054 head->dc_buffer_length = c.dmac_size; 2055 } 2056 2057 /* Link to the previous one if one exists */ 2058 if (chain_kaddr_pre) { 2059 PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT, 2060 chain_paddr); 2061 if (head->dc_next_paddr == 0) 2062 head->dc_next_paddr = chain_paddr; 2063 } 2064 chain_kaddr_pre = chain_kaddr; 2065 2066 /* Maintain pointers */ 2067 chain_paddr += DESC_SIZE; 2068 chain_kaddr += DESC_SIZE; 2069 2070 /* Retrieve the next cookie if there is one */ 2071 if (i < nc-1) 2072 ddi_dma_nextcookie(handle, &c); 2073 } 2074 2075 /* Set the next pointer in the last entry to NULL */ 2076 PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT, 0); 2077 2078 return (DDI_SUCCESS); 2079 } 2080 2081 /* 2082 * Schedule some work. 2083 */ 2084 int 2085 dca_start(dca_t *dca, dca_request_t *reqp, int mcr, int dosched) 2086 { 2087 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2088 2089 mutex_enter(&wlp->dwl_lock); 2090 2091 DBG(dca, DCHATTY, "req=%p, in=%p, out=%p, ctx=%p, ibuf=%p, obuf=%p", 2092 reqp, reqp->dr_in, reqp->dr_out, reqp->dr_ctx_kaddr, 2093 reqp->dr_ibuf_kaddr, reqp->dr_obuf_kaddr); 2094 DBG(dca, DCHATTY, "ctx paddr = %x, ibuf paddr = %x, obuf paddr = %x", 2095 reqp->dr_ctx_paddr, reqp->dr_ibuf_paddr, reqp->dr_obuf_paddr); 2096 /* sync out the entire context and descriptor chains */ 2097 (void) ddi_dma_sync(reqp->dr_ctx_dmah, 0, 0, DDI_DMA_SYNC_FORDEV); 2098 if (dca_check_dma_handle(dca, reqp->dr_ctx_dmah, 2099 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 2100 reqp->destroy = TRUE; 2101 mutex_exit(&wlp->dwl_lock); 2102 return (CRYPTO_DEVICE_ERROR); 2103 } 2104 2105 dca_enqueue(&wlp->dwl_waitq, (dca_listnode_t *)reqp); 2106 wlp->dwl_count++; 2107 wlp->dwl_lastsubmit = ddi_get_lbolt(); 2108 reqp->dr_wlp = wlp; 2109 2110 if ((wlp->dwl_count == wlp->dwl_hiwater) && (wlp->dwl_busy == 0)) { 2111 /* we are fully loaded now, let kCF know */ 2112 2113 wlp->dwl_flowctl++; 2114 wlp->dwl_busy = 1; 2115 2116 crypto_prov_notify(wlp->dwl_prov, CRYPTO_PROVIDER_BUSY); 2117 } 2118 2119 if (dosched) { 2120 #ifdef SCHEDDELAY 2121 /* possibly wait for more work to arrive */ 2122 if (wlp->dwl_count >= wlp->dwl_reqspermcr) { 2123 dca_schedule(dca, mcr); 2124 } else if (!wlp->dwl_schedtid) { 2125 /* wait 1 msec for more work before doing it */ 2126 wlp->dwl_schedtid = timeout(dca_schedtimeout, 2127 (void *)wlp, drv_usectohz(MSEC)); 2128 } 2129 #else 2130 dca_schedule(dca, mcr); 2131 #endif 2132 } 2133 mutex_exit(&wlp->dwl_lock); 2134 2135 return (CRYPTO_QUEUED); 2136 } 2137 2138 void 2139 dca_schedule(dca_t *dca, int mcr) 2140 { 2141 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2142 int csr; 2143 int full; 2144 uint32_t status; 2145 2146 ASSERT(mutex_owned(&wlp->dwl_lock)); 2147 /* 2148 * If the card is draining or has an outstanding failure, 2149 * don't schedule any more work on it right now 2150 */ 2151 if (wlp->dwl_drain || (dca->dca_flags & DCA_FAILED)) { 2152 return; 2153 } 2154 2155 if (mcr == MCR2) { 2156 csr = CSR_MCR2; 2157 full = DMASTAT_MCR2FULL; 2158 } else { 2159 csr = CSR_MCR1; 2160 full = DMASTAT_MCR1FULL; 2161 } 2162 2163 for (;;) { 2164 dca_work_t *workp; 2165 uint32_t offset; 2166 int nreqs; 2167 2168 status = GETCSR(dca, CSR_DMASTAT); 2169 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 2170 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) 2171 return; 2172 2173 if ((status & full) != 0) 2174 break; 2175 2176 #ifdef SCHEDDELAY 2177 /* if there isn't enough to do, don't bother now */ 2178 if ((wlp->dwl_count < wlp->dwl_reqspermcr) && 2179 (ddi_get_lbolt() < (wlp->dwl_lastsubmit + 2180 drv_usectohz(MSEC)))) { 2181 /* wait a bit longer... */ 2182 if (wlp->dwl_schedtid == 0) { 2183 wlp->dwl_schedtid = timeout(dca_schedtimeout, 2184 (void *)wlp, drv_usectohz(MSEC)); 2185 } 2186 return; 2187 } 2188 #endif 2189 2190 /* grab a work structure */ 2191 workp = dca_getwork(dca, mcr); 2192 2193 if (workp == NULL) { 2194 /* 2195 * There must be work ready to be reclaimed, 2196 * in this case, since the chip can only hold 2197 * less work outstanding than there are total. 2198 */ 2199 dca_reclaim(dca, mcr); 2200 continue; 2201 } 2202 2203 nreqs = 0; 2204 offset = MCR_CTXADDR; 2205 2206 while (nreqs < wlp->dwl_reqspermcr) { 2207 dca_request_t *reqp; 2208 2209 reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_waitq); 2210 if (reqp == NULL) { 2211 /* nothing left to process */ 2212 break; 2213 } 2214 /* 2215 * Update flow control. 2216 */ 2217 wlp->dwl_count--; 2218 if ((wlp->dwl_count == wlp->dwl_lowater) && 2219 (wlp->dwl_busy)) { 2220 wlp->dwl_busy = 0; 2221 crypto_prov_notify(wlp->dwl_prov, 2222 CRYPTO_PROVIDER_READY); 2223 } 2224 2225 /* 2226 * Context address. 2227 */ 2228 PUTMCR32(workp, offset, reqp->dr_ctx_paddr); 2229 offset += 4; 2230 2231 /* 2232 * Input chain. 2233 */ 2234 /* input buffer address */ 2235 PUTMCR32(workp, offset, reqp->dr_in_paddr); 2236 offset += 4; 2237 /* next input buffer entry */ 2238 PUTMCR32(workp, offset, reqp->dr_in_next); 2239 offset += 4; 2240 /* input buffer length */ 2241 PUTMCR16(workp, offset, reqp->dr_in_len); 2242 offset += 2; 2243 /* zero the reserved field */ 2244 PUTMCR16(workp, offset, 0); 2245 offset += 2; 2246 2247 /* 2248 * Overall length. 2249 */ 2250 /* reserved field */ 2251 PUTMCR16(workp, offset, 0); 2252 offset += 2; 2253 /* total packet length */ 2254 PUTMCR16(workp, offset, reqp->dr_pkt_length); 2255 offset += 2; 2256 2257 /* 2258 * Output chain. 2259 */ 2260 /* output buffer address */ 2261 PUTMCR32(workp, offset, reqp->dr_out_paddr); 2262 offset += 4; 2263 /* next output buffer entry */ 2264 PUTMCR32(workp, offset, reqp->dr_out_next); 2265 offset += 4; 2266 /* output buffer length */ 2267 PUTMCR16(workp, offset, reqp->dr_out_len); 2268 offset += 2; 2269 /* zero the reserved field */ 2270 PUTMCR16(workp, offset, 0); 2271 offset += 2; 2272 2273 /* 2274 * Note submission. 2275 */ 2276 workp->dw_reqs[nreqs] = reqp; 2277 nreqs++; 2278 } 2279 2280 if (nreqs == 0) { 2281 /* nothing in the queue! */ 2282 dca_freework(workp); 2283 return; 2284 } 2285 2286 wlp->dwl_submit++; 2287 2288 PUTMCR16(workp, MCR_FLAGS, 0); 2289 PUTMCR16(workp, MCR_COUNT, nreqs); 2290 2291 DBG(dca, DCHATTY, 2292 "posting work (phys %x, virt 0x%p) (%d reqs) to MCR%d", 2293 workp->dw_mcr_paddr, workp->dw_mcr_kaddr, 2294 nreqs, mcr); 2295 2296 workp->dw_lbolt = ddi_get_lbolt(); 2297 /* Make sure MCR is synced out to device. */ 2298 (void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 0, 2299 DDI_DMA_SYNC_FORDEV); 2300 if (dca_check_dma_handle(dca, workp->dw_mcr_dmah, 2301 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 2302 dca_destroywork(workp); 2303 return; 2304 } 2305 2306 PUTCSR(dca, csr, workp->dw_mcr_paddr); 2307 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 2308 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 2309 dca_destroywork(workp); 2310 return; 2311 } else { 2312 dca_enqueue(&wlp->dwl_runq, (dca_listnode_t *)workp); 2313 } 2314 2315 DBG(dca, DCHATTY, "posted"); 2316 } 2317 } 2318 2319 /* 2320 * Reclaim completed work, called in interrupt context. 2321 */ 2322 void 2323 dca_reclaim(dca_t *dca, int mcr) 2324 { 2325 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2326 dca_work_t *workp; 2327 ushort_t flags; 2328 int nreclaimed = 0; 2329 int i; 2330 2331 DBG(dca, DRECLAIM, "worklist = 0x%p (MCR%d)", wlp, mcr); 2332 ASSERT(mutex_owned(&wlp->dwl_lock)); 2333 /* 2334 * For each MCR in the submitted (runq), we check to see if 2335 * it has been processed. If so, then we note each individual 2336 * job in the MCR, and and do the completion processing for 2337 * each of such job. 2338 */ 2339 for (;;) { 2340 2341 workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq); 2342 if (workp == NULL) { 2343 break; 2344 } 2345 2346 /* only sync the MCR flags, since that's all we need */ 2347 (void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 4, 2348 DDI_DMA_SYNC_FORKERNEL); 2349 if (dca_check_dma_handle(dca, workp->dw_mcr_dmah, 2350 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 2351 dca_rmqueue((dca_listnode_t *)workp); 2352 dca_destroywork(workp); 2353 return; 2354 } 2355 2356 flags = GETMCR16(workp, MCR_FLAGS); 2357 if ((flags & MCRFLAG_FINISHED) == 0) { 2358 /* chip is still working on it */ 2359 DBG(dca, DRECLAIM, 2360 "chip still working on it (MCR%d)", mcr); 2361 break; 2362 } 2363 2364 /* its really for us, so remove it from the queue */ 2365 dca_rmqueue((dca_listnode_t *)workp); 2366 2367 /* if we were draining, signal on the cv */ 2368 if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) { 2369 cv_signal(&wlp->dwl_cv); 2370 } 2371 2372 /* update statistics, done under the lock */ 2373 for (i = 0; i < wlp->dwl_reqspermcr; i++) { 2374 dca_request_t *reqp = workp->dw_reqs[i]; 2375 if (reqp == NULL) { 2376 continue; 2377 } 2378 if (reqp->dr_byte_stat >= 0) { 2379 dca->dca_stats[reqp->dr_byte_stat] += 2380 reqp->dr_pkt_length; 2381 } 2382 if (reqp->dr_job_stat >= 0) { 2383 dca->dca_stats[reqp->dr_job_stat]++; 2384 } 2385 } 2386 mutex_exit(&wlp->dwl_lock); 2387 2388 for (i = 0; i < wlp->dwl_reqspermcr; i++) { 2389 dca_request_t *reqp = workp->dw_reqs[i]; 2390 2391 if (reqp == NULL) { 2392 continue; 2393 } 2394 2395 /* Do the callback. */ 2396 workp->dw_reqs[i] = NULL; 2397 dca_done(reqp, CRYPTO_SUCCESS); 2398 2399 nreclaimed++; 2400 } 2401 2402 /* now we can release the work */ 2403 dca_freework(workp); 2404 2405 mutex_enter(&wlp->dwl_lock); 2406 } 2407 DBG(dca, DRECLAIM, "reclaimed %d cmds", nreclaimed); 2408 } 2409 2410 int 2411 dca_length(crypto_data_t *cdata) 2412 { 2413 return (cdata->cd_length); 2414 } 2415 2416 /* 2417 * This is the callback function called from the interrupt when a kCF job 2418 * completes. It does some driver-specific things, and then calls the 2419 * kCF-provided callback. Finally, it cleans up the state for the work 2420 * request and drops the reference count to allow for DR. 2421 */ 2422 void 2423 dca_done(dca_request_t *reqp, int err) 2424 { 2425 uint64_t ena = 0; 2426 2427 /* unbind any chains we were using */ 2428 if (dca_unbindchains(reqp) != DDI_SUCCESS) { 2429 /* DMA failure */ 2430 ena = dca_ena(ena); 2431 dca_failure(reqp->dr_dca, DDI_DATAPATH_FAULT, 2432 DCA_FM_ECLASS_NONE, ena, CRYPTO_DEVICE_ERROR, 2433 "fault on buffer DMA handle"); 2434 if (err == CRYPTO_SUCCESS) { 2435 err = CRYPTO_DEVICE_ERROR; 2436 } 2437 } 2438 2439 if (reqp->dr_callback != NULL) { 2440 reqp->dr_callback(reqp, err); 2441 } else { 2442 dca_freereq(reqp); 2443 } 2444 } 2445 2446 /* 2447 * Call this when a failure is detected. It will reset the chip, 2448 * log a message, alert kCF, and mark jobs in the runq as failed. 2449 */ 2450 /* ARGSUSED */ 2451 void 2452 dca_failure(dca_t *dca, ddi_fault_location_t loc, dca_fma_eclass_t index, 2453 uint64_t ena, int errno, char *mess, ...) 2454 { 2455 va_list ap; 2456 char buf[256]; 2457 int mcr; 2458 char *eclass; 2459 int have_mutex; 2460 2461 va_start(ap, mess); 2462 (void) vsprintf(buf, mess, ap); 2463 va_end(ap); 2464 2465 eclass = dca_fma_eclass_string(dca->dca_model, index); 2466 2467 if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) && 2468 index != DCA_FM_ECLASS_NONE) { 2469 ddi_fm_ereport_post(dca->dca_dip, eclass, ena, 2470 DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8, 2471 FM_EREPORT_VERS0, NULL); 2472 2473 /* Report the impact of the failure to the DDI. */ 2474 ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_LOST); 2475 } else { 2476 /* Just log the error string to the message log */ 2477 dca_error(dca, buf); 2478 } 2479 2480 /* 2481 * Indicate a failure (keeps schedule from running). 2482 */ 2483 dca->dca_flags |= DCA_FAILED; 2484 2485 /* 2486 * Reset the chip. This should also have as a side effect, the 2487 * disabling of all interrupts from the device. 2488 */ 2489 (void) dca_reset(dca, 1); 2490 2491 /* 2492 * Report the failure to kCF. 2493 */ 2494 for (mcr = MCR1; mcr <= MCR2; mcr++) { 2495 if (WORKLIST(dca, mcr)->dwl_prov) { 2496 crypto_prov_notify(WORKLIST(dca, mcr)->dwl_prov, 2497 CRYPTO_PROVIDER_FAILED); 2498 } 2499 } 2500 2501 /* 2502 * Return jobs not sent to hardware back to kCF. 2503 */ 2504 dca_rejectjobs(dca); 2505 2506 /* 2507 * From this point on, no new work should be arriving, and the 2508 * chip should not be doing any active DMA. 2509 */ 2510 2511 /* 2512 * Now find all the work submitted to the device and fail 2513 * them. 2514 */ 2515 for (mcr = MCR1; mcr <= MCR2; mcr++) { 2516 dca_worklist_t *wlp; 2517 int i; 2518 2519 wlp = WORKLIST(dca, mcr); 2520 2521 if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) { 2522 continue; 2523 } 2524 for (;;) { 2525 dca_work_t *workp; 2526 2527 have_mutex = mutex_tryenter(&wlp->dwl_lock); 2528 workp = (dca_work_t *)dca_dequeue(&wlp->dwl_runq); 2529 if (workp == NULL) { 2530 if (have_mutex) 2531 mutex_exit(&wlp->dwl_lock); 2532 break; 2533 } 2534 mutex_exit(&wlp->dwl_lock); 2535 2536 /* 2537 * Free up requests 2538 */ 2539 for (i = 0; i < wlp->dwl_reqspermcr; i++) { 2540 dca_request_t *reqp = workp->dw_reqs[i]; 2541 if (reqp) { 2542 dca_done(reqp, errno); 2543 workp->dw_reqs[i] = NULL; 2544 } 2545 } 2546 2547 mutex_enter(&wlp->dwl_lock); 2548 /* 2549 * If waiting to drain, signal on the waiter. 2550 */ 2551 if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) { 2552 cv_signal(&wlp->dwl_cv); 2553 } 2554 2555 /* 2556 * Return the work and request structures to 2557 * the free pool. 2558 */ 2559 dca_freework(workp); 2560 if (have_mutex) 2561 mutex_exit(&wlp->dwl_lock); 2562 } 2563 } 2564 2565 } 2566 2567 #ifdef SCHEDDELAY 2568 /* 2569 * Reschedule worklist as needed. 2570 */ 2571 void 2572 dca_schedtimeout(void *arg) 2573 { 2574 dca_worklist_t *wlp = (dca_worklist_t *)arg; 2575 mutex_enter(&wlp->dwl_lock); 2576 wlp->dwl_schedtid = 0; 2577 dca_schedule(wlp->dwl_dca, wlp->dwl_mcr); 2578 mutex_exit(&wlp->dwl_lock); 2579 } 2580 #endif 2581 2582 /* 2583 * Check for stalled jobs. 2584 */ 2585 void 2586 dca_jobtimeout(void *arg) 2587 { 2588 int mcr; 2589 dca_t *dca = (dca_t *)arg; 2590 int hung = 0; 2591 2592 for (mcr = MCR1; mcr <= MCR2; mcr++) { 2593 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2594 dca_work_t *workp; 2595 clock_t when; 2596 2597 mutex_enter(&wlp->dwl_lock); 2598 when = ddi_get_lbolt(); 2599 2600 workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq); 2601 if (workp == NULL) { 2602 /* nothing sitting in the queue */ 2603 mutex_exit(&wlp->dwl_lock); 2604 continue; 2605 } 2606 2607 if ((when - workp->dw_lbolt) < drv_usectohz(STALETIME)) { 2608 /* request has been queued for less than STALETIME */ 2609 mutex_exit(&wlp->dwl_lock); 2610 continue; 2611 } 2612 2613 /* job has been sitting around for over 1 second, badness */ 2614 DBG(dca, DWARN, "stale job (0x%p) found in MCR%d!", workp, 2615 mcr); 2616 2617 /* put it back in the queue, until we reset the chip */ 2618 hung++; 2619 mutex_exit(&wlp->dwl_lock); 2620 } 2621 2622 if (hung) { 2623 dca_failure(dca, DDI_DEVICE_FAULT, 2624 DCA_FM_ECLASS_HW_TIMEOUT, dca_ena(0), CRYPTO_DEVICE_ERROR, 2625 "timeout processing job.)"); 2626 } 2627 2628 /* reschedule ourself */ 2629 mutex_enter(&dca->dca_intrlock); 2630 if (dca->dca_jobtid == 0) { 2631 /* timeout has been canceled, prior to DR */ 2632 mutex_exit(&dca->dca_intrlock); 2633 return; 2634 } 2635 2636 /* check again in 1 second */ 2637 dca->dca_jobtid = timeout(dca_jobtimeout, arg, 2638 drv_usectohz(SECOND)); 2639 mutex_exit(&dca->dca_intrlock); 2640 } 2641 2642 /* 2643 * This returns all jobs back to kCF. It assumes that processing 2644 * on the worklist has halted. 2645 */ 2646 void 2647 dca_rejectjobs(dca_t *dca) 2648 { 2649 int mcr; 2650 int have_mutex; 2651 for (mcr = MCR1; mcr <= MCR2; mcr++) { 2652 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2653 dca_request_t *reqp; 2654 2655 if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) { 2656 continue; 2657 } 2658 have_mutex = mutex_tryenter(&wlp->dwl_lock); 2659 for (;;) { 2660 reqp = (dca_request_t *)dca_unqueue(&wlp->dwl_waitq); 2661 if (reqp == NULL) { 2662 break; 2663 } 2664 /* update flow control */ 2665 wlp->dwl_count--; 2666 if ((wlp->dwl_count == wlp->dwl_lowater) && 2667 (wlp->dwl_busy)) { 2668 wlp->dwl_busy = 0; 2669 crypto_prov_notify(wlp->dwl_prov, 2670 CRYPTO_PROVIDER_READY); 2671 } 2672 mutex_exit(&wlp->dwl_lock); 2673 2674 (void) dca_unbindchains(reqp); 2675 reqp->dr_callback(reqp, EAGAIN); 2676 mutex_enter(&wlp->dwl_lock); 2677 } 2678 if (have_mutex) 2679 mutex_exit(&wlp->dwl_lock); 2680 } 2681 } 2682 2683 int 2684 dca_drain(dca_t *dca) 2685 { 2686 int mcr; 2687 for (mcr = MCR1; mcr <= MCR2; mcr++) { 2688 #ifdef SCHEDDELAY 2689 timeout_id_t tid; 2690 #endif 2691 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2692 2693 mutex_enter(&wlp->dwl_lock); 2694 wlp->dwl_drain = 1; 2695 2696 /* give it up to a second to drain from the chip */ 2697 if (!QEMPTY(&wlp->dwl_runq)) { 2698 (void) cv_reltimedwait(&wlp->dwl_cv, &wlp->dwl_lock, 2699 drv_usectohz(STALETIME), TR_CLOCK_TICK); 2700 2701 if (!QEMPTY(&wlp->dwl_runq)) { 2702 dca_error(dca, "unable to drain device"); 2703 mutex_exit(&wlp->dwl_lock); 2704 dca_undrain(dca); 2705 return (EBUSY); 2706 } 2707 } 2708 2709 #ifdef SCHEDDELAY 2710 tid = wlp->dwl_schedtid; 2711 mutex_exit(&wlp->dwl_lock); 2712 2713 /* 2714 * untimeout outside the lock -- this is safe because we 2715 * have set the drain flag, so dca_schedule() will not 2716 * reschedule another timeout 2717 */ 2718 if (tid) { 2719 untimeout(tid); 2720 } 2721 #else 2722 mutex_exit(&wlp->dwl_lock); 2723 #endif 2724 } 2725 return (0); 2726 } 2727 2728 void 2729 dca_undrain(dca_t *dca) 2730 { 2731 int mcr; 2732 2733 for (mcr = MCR1; mcr <= MCR2; mcr++) { 2734 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2735 mutex_enter(&wlp->dwl_lock); 2736 wlp->dwl_drain = 0; 2737 dca_schedule(dca, mcr); 2738 mutex_exit(&wlp->dwl_lock); 2739 } 2740 } 2741 2742 /* 2743 * Duplicate the crypto_data_t structure, but point to the original 2744 * buffers. 2745 */ 2746 int 2747 dca_dupcrypto(crypto_data_t *input, crypto_data_t *ninput) 2748 { 2749 ninput->cd_format = input->cd_format; 2750 ninput->cd_offset = input->cd_offset; 2751 ninput->cd_length = input->cd_length; 2752 ninput->cd_miscdata = input->cd_miscdata; 2753 2754 switch (input->cd_format) { 2755 case CRYPTO_DATA_RAW: 2756 ninput->cd_raw.iov_base = input->cd_raw.iov_base; 2757 ninput->cd_raw.iov_len = input->cd_raw.iov_len; 2758 break; 2759 2760 case CRYPTO_DATA_UIO: 2761 ninput->cd_uio = input->cd_uio; 2762 break; 2763 2764 case CRYPTO_DATA_MBLK: 2765 ninput->cd_mp = input->cd_mp; 2766 break; 2767 2768 default: 2769 DBG(NULL, DWARN, 2770 "dca_dupcrypto: unrecognised crypto data format"); 2771 return (CRYPTO_FAILED); 2772 } 2773 2774 return (CRYPTO_SUCCESS); 2775 } 2776 2777 /* 2778 * Performs validation checks on the input and output data structures. 2779 */ 2780 int 2781 dca_verifyio(crypto_data_t *input, crypto_data_t *output) 2782 { 2783 int rv = CRYPTO_SUCCESS; 2784 2785 switch (input->cd_format) { 2786 case CRYPTO_DATA_RAW: 2787 break; 2788 2789 case CRYPTO_DATA_UIO: 2790 /* we support only kernel buffer */ 2791 if (input->cd_uio->uio_segflg != UIO_SYSSPACE) { 2792 DBG(NULL, DWARN, "non kernel input uio buffer"); 2793 rv = CRYPTO_ARGUMENTS_BAD; 2794 } 2795 break; 2796 2797 case CRYPTO_DATA_MBLK: 2798 break; 2799 2800 default: 2801 DBG(NULL, DWARN, "unrecognised input crypto data format"); 2802 rv = CRYPTO_ARGUMENTS_BAD; 2803 } 2804 2805 switch (output->cd_format) { 2806 case CRYPTO_DATA_RAW: 2807 break; 2808 2809 case CRYPTO_DATA_UIO: 2810 /* we support only kernel buffer */ 2811 if (output->cd_uio->uio_segflg != UIO_SYSSPACE) { 2812 DBG(NULL, DWARN, "non kernel output uio buffer"); 2813 rv = CRYPTO_ARGUMENTS_BAD; 2814 } 2815 break; 2816 2817 case CRYPTO_DATA_MBLK: 2818 break; 2819 2820 default: 2821 DBG(NULL, DWARN, "unrecognised output crypto data format"); 2822 rv = CRYPTO_ARGUMENTS_BAD; 2823 } 2824 2825 return (rv); 2826 } 2827 2828 /* 2829 * data: source crypto_data_t struct 2830 * off: offset into the source before commencing copy 2831 * count: the amount of data to copy 2832 * dest: destination buffer 2833 */ 2834 int 2835 dca_getbufbytes(crypto_data_t *data, size_t off, int count, uchar_t *dest) 2836 { 2837 int rv = CRYPTO_SUCCESS; 2838 uio_t *uiop; 2839 uint_t vec_idx; 2840 size_t cur_len; 2841 mblk_t *mp; 2842 2843 if (count == 0) { 2844 /* We don't want anything so we're done. */ 2845 return (rv); 2846 } 2847 2848 /* 2849 * Sanity check that we haven't specified a length greater than the 2850 * offset adjusted size of the buffer. 2851 */ 2852 if (count > (data->cd_length - off)) { 2853 return (CRYPTO_DATA_LEN_RANGE); 2854 } 2855 2856 /* Add the internal crypto_data offset to the requested offset. */ 2857 off += data->cd_offset; 2858 2859 switch (data->cd_format) { 2860 case CRYPTO_DATA_RAW: 2861 bcopy(data->cd_raw.iov_base + off, dest, count); 2862 break; 2863 2864 case CRYPTO_DATA_UIO: 2865 /* 2866 * Jump to the first iovec containing data to be 2867 * processed. 2868 */ 2869 uiop = data->cd_uio; 2870 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && 2871 off >= uiop->uio_iov[vec_idx].iov_len; 2872 off -= uiop->uio_iov[vec_idx++].iov_len) 2873 ; 2874 if (vec_idx == uiop->uio_iovcnt) { 2875 /* 2876 * The caller specified an offset that is larger than 2877 * the total size of the buffers it provided. 2878 */ 2879 return (CRYPTO_DATA_LEN_RANGE); 2880 } 2881 2882 /* 2883 * Now process the iovecs. 2884 */ 2885 while (vec_idx < uiop->uio_iovcnt && count > 0) { 2886 cur_len = min(uiop->uio_iov[vec_idx].iov_len - 2887 off, count); 2888 bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest, 2889 cur_len); 2890 count -= cur_len; 2891 dest += cur_len; 2892 vec_idx++; 2893 off = 0; 2894 } 2895 2896 if (vec_idx == uiop->uio_iovcnt && count > 0) { 2897 /* 2898 * The end of the specified iovec's was reached but 2899 * the length requested could not be processed 2900 * (requested to digest more data than it provided). 2901 */ 2902 return (CRYPTO_DATA_LEN_RANGE); 2903 } 2904 break; 2905 2906 case CRYPTO_DATA_MBLK: 2907 /* 2908 * Jump to the first mblk_t containing data to be processed. 2909 */ 2910 for (mp = data->cd_mp; mp != NULL && off >= MBLKL(mp); 2911 off -= MBLKL(mp), mp = mp->b_cont) 2912 ; 2913 if (mp == NULL) { 2914 /* 2915 * The caller specified an offset that is larger than 2916 * the total size of the buffers it provided. 2917 */ 2918 return (CRYPTO_DATA_LEN_RANGE); 2919 } 2920 2921 /* 2922 * Now do the processing on the mblk chain. 2923 */ 2924 while (mp != NULL && count > 0) { 2925 cur_len = min(MBLKL(mp) - off, count); 2926 bcopy((char *)(mp->b_rptr + off), dest, cur_len); 2927 count -= cur_len; 2928 dest += cur_len; 2929 mp = mp->b_cont; 2930 off = 0; 2931 } 2932 2933 if (mp == NULL && count > 0) { 2934 /* 2935 * The end of the mblk was reached but the length 2936 * requested could not be processed, (requested to 2937 * digest more data than it provided). 2938 */ 2939 return (CRYPTO_DATA_LEN_RANGE); 2940 } 2941 break; 2942 2943 default: 2944 DBG(NULL, DWARN, "unrecognised crypto data format"); 2945 rv = CRYPTO_ARGUMENTS_BAD; 2946 } 2947 return (rv); 2948 } 2949 2950 2951 /* 2952 * Performs the input, output or hard scatter/gather checks on the specified 2953 * crypto_data_t struct. Returns true if the data is scatter/gather in nature 2954 * ie fails the test. 2955 */ 2956 int 2957 dca_sgcheck(dca_t *dca, crypto_data_t *data, dca_sg_param_t val) 2958 { 2959 uio_t *uiop; 2960 mblk_t *mp; 2961 int rv = FALSE; 2962 2963 switch (val) { 2964 case DCA_SG_CONTIG: 2965 /* 2966 * Check for a contiguous data buffer. 2967 */ 2968 switch (data->cd_format) { 2969 case CRYPTO_DATA_RAW: 2970 /* Contiguous in nature */ 2971 break; 2972 2973 case CRYPTO_DATA_UIO: 2974 if (data->cd_uio->uio_iovcnt > 1) 2975 rv = TRUE; 2976 break; 2977 2978 case CRYPTO_DATA_MBLK: 2979 mp = data->cd_mp; 2980 if (mp->b_cont != NULL) 2981 rv = TRUE; 2982 break; 2983 2984 default: 2985 DBG(NULL, DWARN, "unrecognised crypto data format"); 2986 } 2987 break; 2988 2989 case DCA_SG_WALIGN: 2990 /* 2991 * Check for a contiguous data buffer that is 32-bit word 2992 * aligned and is of word multiples in size. 2993 */ 2994 switch (data->cd_format) { 2995 case CRYPTO_DATA_RAW: 2996 if ((data->cd_raw.iov_len % sizeof (uint32_t)) || 2997 ((uintptr_t)data->cd_raw.iov_base % 2998 sizeof (uint32_t))) { 2999 rv = TRUE; 3000 } 3001 break; 3002 3003 case CRYPTO_DATA_UIO: 3004 uiop = data->cd_uio; 3005 if (uiop->uio_iovcnt > 1) { 3006 return (TRUE); 3007 } 3008 /* So there is only one iovec */ 3009 if ((uiop->uio_iov[0].iov_len % sizeof (uint32_t)) || 3010 ((uintptr_t)uiop->uio_iov[0].iov_base % 3011 sizeof (uint32_t))) { 3012 rv = TRUE; 3013 } 3014 break; 3015 3016 case CRYPTO_DATA_MBLK: 3017 mp = data->cd_mp; 3018 if (mp->b_cont != NULL) { 3019 return (TRUE); 3020 } 3021 /* So there is only one mblk in the chain */ 3022 if ((MBLKL(mp) % sizeof (uint32_t)) || 3023 ((uintptr_t)mp->b_rptr % sizeof (uint32_t))) { 3024 rv = TRUE; 3025 } 3026 break; 3027 3028 default: 3029 DBG(NULL, DWARN, "unrecognised crypto data format"); 3030 } 3031 break; 3032 3033 case DCA_SG_PALIGN: 3034 /* 3035 * Check that the data buffer is page aligned and is of 3036 * page multiples in size. 3037 */ 3038 switch (data->cd_format) { 3039 case CRYPTO_DATA_RAW: 3040 if ((data->cd_length % dca->dca_pagesize) || 3041 ((uintptr_t)data->cd_raw.iov_base % 3042 dca->dca_pagesize)) { 3043 rv = TRUE; 3044 } 3045 break; 3046 3047 case CRYPTO_DATA_UIO: 3048 uiop = data->cd_uio; 3049 if ((uiop->uio_iov[0].iov_len % dca->dca_pagesize) || 3050 ((uintptr_t)uiop->uio_iov[0].iov_base % 3051 dca->dca_pagesize)) { 3052 rv = TRUE; 3053 } 3054 break; 3055 3056 case CRYPTO_DATA_MBLK: 3057 mp = data->cd_mp; 3058 if ((MBLKL(mp) % dca->dca_pagesize) || 3059 ((uintptr_t)mp->b_rptr % dca->dca_pagesize)) { 3060 rv = TRUE; 3061 } 3062 break; 3063 3064 default: 3065 DBG(NULL, DWARN, "unrecognised crypto data format"); 3066 } 3067 break; 3068 3069 default: 3070 DBG(NULL, DWARN, "unrecognised scatter/gather param type"); 3071 } 3072 3073 return (rv); 3074 } 3075 3076 /* 3077 * Increments the cd_offset and decrements the cd_length as the data is 3078 * gathered from the crypto_data_t struct. 3079 * The data is reverse-copied into the dest buffer if the flag is true. 3080 */ 3081 int 3082 dca_gather(crypto_data_t *in, char *dest, int count, int reverse) 3083 { 3084 int rv = CRYPTO_SUCCESS; 3085 uint_t vec_idx; 3086 uio_t *uiop; 3087 off_t off = in->cd_offset; 3088 size_t cur_len; 3089 mblk_t *mp; 3090 3091 switch (in->cd_format) { 3092 case CRYPTO_DATA_RAW: 3093 if (count > in->cd_length) { 3094 /* 3095 * The caller specified a length greater than the 3096 * size of the buffer. 3097 */ 3098 return (CRYPTO_DATA_LEN_RANGE); 3099 } 3100 if (reverse) 3101 dca_reverse(in->cd_raw.iov_base + off, dest, count, 3102 count); 3103 else 3104 bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count); 3105 in->cd_offset += count; 3106 in->cd_length -= count; 3107 break; 3108 3109 case CRYPTO_DATA_UIO: 3110 /* 3111 * Jump to the first iovec containing data to be processed. 3112 */ 3113 uiop = in->cd_uio; 3114 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && 3115 off >= uiop->uio_iov[vec_idx].iov_len; 3116 off -= uiop->uio_iov[vec_idx++].iov_len) 3117 ; 3118 if (vec_idx == uiop->uio_iovcnt) { 3119 /* 3120 * The caller specified an offset that is larger than 3121 * the total size of the buffers it provided. 3122 */ 3123 return (CRYPTO_DATA_LEN_RANGE); 3124 } 3125 3126 /* 3127 * Now process the iovecs. 3128 */ 3129 while (vec_idx < uiop->uio_iovcnt && count > 0) { 3130 cur_len = min(uiop->uio_iov[vec_idx].iov_len - 3131 off, count); 3132 count -= cur_len; 3133 if (reverse) { 3134 /* Fill the dest buffer from the end */ 3135 dca_reverse(uiop->uio_iov[vec_idx].iov_base + 3136 off, dest+count, cur_len, cur_len); 3137 } else { 3138 bcopy(uiop->uio_iov[vec_idx].iov_base + off, 3139 dest, cur_len); 3140 dest += cur_len; 3141 } 3142 in->cd_offset += cur_len; 3143 in->cd_length -= cur_len; 3144 vec_idx++; 3145 off = 0; 3146 } 3147 3148 if (vec_idx == uiop->uio_iovcnt && count > 0) { 3149 /* 3150 * The end of the specified iovec's was reached but 3151 * the length requested could not be processed 3152 * (requested to digest more data than it provided). 3153 */ 3154 return (CRYPTO_DATA_LEN_RANGE); 3155 } 3156 break; 3157 3158 case CRYPTO_DATA_MBLK: 3159 /* 3160 * Jump to the first mblk_t containing data to be processed. 3161 */ 3162 for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp); 3163 off -= MBLKL(mp), mp = mp->b_cont) 3164 ; 3165 if (mp == NULL) { 3166 /* 3167 * The caller specified an offset that is larger than 3168 * the total size of the buffers it provided. 3169 */ 3170 return (CRYPTO_DATA_LEN_RANGE); 3171 } 3172 3173 /* 3174 * Now do the processing on the mblk chain. 3175 */ 3176 while (mp != NULL && count > 0) { 3177 cur_len = min(MBLKL(mp) - off, count); 3178 count -= cur_len; 3179 if (reverse) { 3180 /* Fill the dest buffer from the end */ 3181 dca_reverse((char *)(mp->b_rptr + off), 3182 dest+count, cur_len, cur_len); 3183 } else { 3184 bcopy((char *)(mp->b_rptr + off), dest, 3185 cur_len); 3186 dest += cur_len; 3187 } 3188 in->cd_offset += cur_len; 3189 in->cd_length -= cur_len; 3190 mp = mp->b_cont; 3191 off = 0; 3192 } 3193 3194 if (mp == NULL && count > 0) { 3195 /* 3196 * The end of the mblk was reached but the length 3197 * requested could not be processed, (requested to 3198 * digest more data than it provided). 3199 */ 3200 return (CRYPTO_DATA_LEN_RANGE); 3201 } 3202 break; 3203 3204 default: 3205 DBG(NULL, DWARN, "dca_gather: unrecognised crypto data format"); 3206 rv = CRYPTO_ARGUMENTS_BAD; 3207 } 3208 return (rv); 3209 } 3210 3211 /* 3212 * Increments the cd_offset and decrements the cd_length as the data is 3213 * gathered from the crypto_data_t struct. 3214 */ 3215 int 3216 dca_resid_gather(crypto_data_t *in, char *resid, int *residlen, char *dest, 3217 int count) 3218 { 3219 int rv = CRYPTO_SUCCESS; 3220 caddr_t baddr; 3221 uint_t vec_idx; 3222 uio_t *uiop; 3223 off_t off = in->cd_offset; 3224 size_t cur_len; 3225 mblk_t *mp; 3226 3227 /* Process the residual first */ 3228 if (*residlen > 0) { 3229 uint_t num = min(count, *residlen); 3230 bcopy(resid, dest, num); 3231 *residlen -= num; 3232 if (*residlen > 0) { 3233 /* 3234 * Requested amount 'count' is less than what's in 3235 * the residual, so shuffle any remaining resid to 3236 * the front. 3237 */ 3238 baddr = resid + num; 3239 bcopy(baddr, resid, *residlen); 3240 } 3241 dest += num; 3242 count -= num; 3243 } 3244 3245 /* Now process what's in the crypto_data_t structs */ 3246 switch (in->cd_format) { 3247 case CRYPTO_DATA_RAW: 3248 if (count > in->cd_length) { 3249 /* 3250 * The caller specified a length greater than the 3251 * size of the buffer. 3252 */ 3253 return (CRYPTO_DATA_LEN_RANGE); 3254 } 3255 bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count); 3256 in->cd_offset += count; 3257 in->cd_length -= count; 3258 break; 3259 3260 case CRYPTO_DATA_UIO: 3261 /* 3262 * Jump to the first iovec containing data to be processed. 3263 */ 3264 uiop = in->cd_uio; 3265 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && 3266 off >= uiop->uio_iov[vec_idx].iov_len; 3267 off -= uiop->uio_iov[vec_idx++].iov_len) 3268 ; 3269 if (vec_idx == uiop->uio_iovcnt) { 3270 /* 3271 * The caller specified an offset that is larger than 3272 * the total size of the buffers it provided. 3273 */ 3274 return (CRYPTO_DATA_LEN_RANGE); 3275 } 3276 3277 /* 3278 * Now process the iovecs. 3279 */ 3280 while (vec_idx < uiop->uio_iovcnt && count > 0) { 3281 cur_len = min(uiop->uio_iov[vec_idx].iov_len - 3282 off, count); 3283 bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest, 3284 cur_len); 3285 count -= cur_len; 3286 dest += cur_len; 3287 in->cd_offset += cur_len; 3288 in->cd_length -= cur_len; 3289 vec_idx++; 3290 off = 0; 3291 } 3292 3293 if (vec_idx == uiop->uio_iovcnt && count > 0) { 3294 /* 3295 * The end of the specified iovec's was reached but 3296 * the length requested could not be processed 3297 * (requested to digest more data than it provided). 3298 */ 3299 return (CRYPTO_DATA_LEN_RANGE); 3300 } 3301 break; 3302 3303 case CRYPTO_DATA_MBLK: 3304 /* 3305 * Jump to the first mblk_t containing data to be processed. 3306 */ 3307 for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp); 3308 off -= MBLKL(mp), mp = mp->b_cont) 3309 ; 3310 if (mp == NULL) { 3311 /* 3312 * The caller specified an offset that is larger than 3313 * the total size of the buffers it provided. 3314 */ 3315 return (CRYPTO_DATA_LEN_RANGE); 3316 } 3317 3318 /* 3319 * Now do the processing on the mblk chain. 3320 */ 3321 while (mp != NULL && count > 0) { 3322 cur_len = min(MBLKL(mp) - off, count); 3323 bcopy((char *)(mp->b_rptr + off), dest, cur_len); 3324 count -= cur_len; 3325 dest += cur_len; 3326 in->cd_offset += cur_len; 3327 in->cd_length -= cur_len; 3328 mp = mp->b_cont; 3329 off = 0; 3330 } 3331 3332 if (mp == NULL && count > 0) { 3333 /* 3334 * The end of the mblk was reached but the length 3335 * requested could not be processed, (requested to 3336 * digest more data than it provided). 3337 */ 3338 return (CRYPTO_DATA_LEN_RANGE); 3339 } 3340 break; 3341 3342 default: 3343 DBG(NULL, DWARN, 3344 "dca_resid_gather: unrecognised crypto data format"); 3345 rv = CRYPTO_ARGUMENTS_BAD; 3346 } 3347 return (rv); 3348 } 3349 3350 /* 3351 * Appends the data to the crypto_data_t struct increasing cd_length. 3352 * cd_offset is left unchanged. 3353 * Data is reverse-copied if the flag is TRUE. 3354 */ 3355 int 3356 dca_scatter(const char *src, crypto_data_t *out, int count, int reverse) 3357 { 3358 int rv = CRYPTO_SUCCESS; 3359 off_t offset = out->cd_offset + out->cd_length; 3360 uint_t vec_idx; 3361 uio_t *uiop; 3362 size_t cur_len; 3363 mblk_t *mp; 3364 3365 switch (out->cd_format) { 3366 case CRYPTO_DATA_RAW: 3367 if (out->cd_raw.iov_len - offset < count) { 3368 /* Trying to write out more than space available. */ 3369 return (CRYPTO_DATA_LEN_RANGE); 3370 } 3371 if (reverse) 3372 dca_reverse((void*) src, out->cd_raw.iov_base + offset, 3373 count, count); 3374 else 3375 bcopy(src, out->cd_raw.iov_base + offset, count); 3376 out->cd_length += count; 3377 break; 3378 3379 case CRYPTO_DATA_UIO: 3380 /* 3381 * Jump to the first iovec that can be written to. 3382 */ 3383 uiop = out->cd_uio; 3384 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && 3385 offset >= uiop->uio_iov[vec_idx].iov_len; 3386 offset -= uiop->uio_iov[vec_idx++].iov_len) 3387 ; 3388 if (vec_idx == uiop->uio_iovcnt) { 3389 /* 3390 * The caller specified an offset that is larger than 3391 * the total size of the buffers it provided. 3392 */ 3393 return (CRYPTO_DATA_LEN_RANGE); 3394 } 3395 3396 /* 3397 * Now process the iovecs. 3398 */ 3399 while (vec_idx < uiop->uio_iovcnt && count > 0) { 3400 cur_len = min(uiop->uio_iov[vec_idx].iov_len - 3401 offset, count); 3402 count -= cur_len; 3403 if (reverse) { 3404 dca_reverse((void*) (src+count), 3405 uiop->uio_iov[vec_idx].iov_base + 3406 offset, cur_len, cur_len); 3407 } else { 3408 bcopy(src, uiop->uio_iov[vec_idx].iov_base + 3409 offset, cur_len); 3410 src += cur_len; 3411 } 3412 out->cd_length += cur_len; 3413 vec_idx++; 3414 offset = 0; 3415 } 3416 3417 if (vec_idx == uiop->uio_iovcnt && count > 0) { 3418 /* 3419 * The end of the specified iovec's was reached but 3420 * the length requested could not be processed 3421 * (requested to write more data than space provided). 3422 */ 3423 return (CRYPTO_DATA_LEN_RANGE); 3424 } 3425 break; 3426 3427 case CRYPTO_DATA_MBLK: 3428 /* 3429 * Jump to the first mblk_t that can be written to. 3430 */ 3431 for (mp = out->cd_mp; mp != NULL && offset >= MBLKL(mp); 3432 offset -= MBLKL(mp), mp = mp->b_cont) 3433 ; 3434 if (mp == NULL) { 3435 /* 3436 * The caller specified an offset that is larger than 3437 * the total size of the buffers it provided. 3438 */ 3439 return (CRYPTO_DATA_LEN_RANGE); 3440 } 3441 3442 /* 3443 * Now do the processing on the mblk chain. 3444 */ 3445 while (mp != NULL && count > 0) { 3446 cur_len = min(MBLKL(mp) - offset, count); 3447 count -= cur_len; 3448 if (reverse) { 3449 dca_reverse((void*) (src+count), 3450 (char *)(mp->b_rptr + offset), cur_len, 3451 cur_len); 3452 } else { 3453 bcopy(src, (char *)(mp->b_rptr + offset), 3454 cur_len); 3455 src += cur_len; 3456 } 3457 out->cd_length += cur_len; 3458 mp = mp->b_cont; 3459 offset = 0; 3460 } 3461 3462 if (mp == NULL && count > 0) { 3463 /* 3464 * The end of the mblk was reached but the length 3465 * requested could not be processed, (requested to 3466 * digest more data than it provided). 3467 */ 3468 return (CRYPTO_DATA_LEN_RANGE); 3469 } 3470 break; 3471 3472 default: 3473 DBG(NULL, DWARN, "unrecognised crypto data format"); 3474 rv = CRYPTO_ARGUMENTS_BAD; 3475 } 3476 return (rv); 3477 } 3478 3479 /* 3480 * Compare two byte arrays in reverse order. 3481 * Return 0 if they are identical, 1 otherwise. 3482 */ 3483 int 3484 dca_bcmp_reverse(const void *s1, const void *s2, size_t n) 3485 { 3486 int i; 3487 caddr_t src, dst; 3488 3489 if (!n) 3490 return (0); 3491 3492 src = ((caddr_t)s1) + n - 1; 3493 dst = (caddr_t)s2; 3494 for (i = 0; i < n; i++) { 3495 if (*src != *dst) 3496 return (1); 3497 src--; 3498 dst++; 3499 } 3500 3501 return (0); 3502 } 3503 3504 3505 /* 3506 * This calculates the size of a bignum in bits, specifically not counting 3507 * leading zero bits. This size calculation must be done *before* any 3508 * endian reversal takes place (i.e. the numbers are in absolute big-endian 3509 * order.) 3510 */ 3511 int 3512 dca_bitlen(unsigned char *bignum, int bytelen) 3513 { 3514 unsigned char msbyte; 3515 int i, j; 3516 3517 for (i = 0; i < bytelen - 1; i++) { 3518 if (bignum[i] != 0) { 3519 break; 3520 } 3521 } 3522 msbyte = bignum[i]; 3523 for (j = 8; j > 1; j--) { 3524 if (msbyte & 0x80) { 3525 break; 3526 } 3527 msbyte <<= 1; 3528 } 3529 return ((8 * (bytelen - i - 1)) + j); 3530 } 3531 3532 /* 3533 * This compares to bignums (in big-endian order). It ignores leading 3534 * null bytes. The result semantics follow bcmp, mempcmp, strcmp, etc. 3535 */ 3536 int 3537 dca_numcmp(caddr_t n1, int n1len, caddr_t n2, int n2len) 3538 { 3539 while ((n1len > 1) && (*n1 == 0)) { 3540 n1len--; 3541 n1++; 3542 } 3543 while ((n2len > 1) && (*n2 == 0)) { 3544 n2len--; 3545 n2++; 3546 } 3547 if (n1len != n2len) { 3548 return (n1len - n2len); 3549 } 3550 while ((n1len > 1) && (*n1 == *n2)) { 3551 n1++; 3552 n2++; 3553 n1len--; 3554 } 3555 return ((int)(*(uchar_t *)n1) - (int)(*(uchar_t *)n2)); 3556 } 3557 3558 /* 3559 * Return array of key attributes. 3560 */ 3561 crypto_object_attribute_t * 3562 dca_get_key_attr(crypto_key_t *key) 3563 { 3564 if ((key->ck_format != CRYPTO_KEY_ATTR_LIST) || 3565 (key->ck_count == 0)) { 3566 return (NULL); 3567 } 3568 3569 return (key->ck_attrs); 3570 } 3571 3572 /* 3573 * If attribute type exists valp points to it's 32-bit value. 3574 */ 3575 int 3576 dca_attr_lookup_uint32(crypto_object_attribute_t *attrp, uint_t atnum, 3577 uint64_t atype, uint32_t *valp) 3578 { 3579 crypto_object_attribute_t *bap; 3580 3581 bap = dca_find_attribute(attrp, atnum, atype); 3582 if (bap == NULL) { 3583 return (CRYPTO_ATTRIBUTE_TYPE_INVALID); 3584 } 3585 3586 *valp = *bap->oa_value; 3587 3588 return (CRYPTO_SUCCESS); 3589 } 3590 3591 /* 3592 * If attribute type exists data contains the start address of the value, 3593 * and numelems contains it's length. 3594 */ 3595 int 3596 dca_attr_lookup_uint8_array(crypto_object_attribute_t *attrp, uint_t atnum, 3597 uint64_t atype, void **data, unsigned int *numelems) 3598 { 3599 crypto_object_attribute_t *bap; 3600 3601 bap = dca_find_attribute(attrp, atnum, atype); 3602 if (bap == NULL) { 3603 return (CRYPTO_ATTRIBUTE_TYPE_INVALID); 3604 } 3605 3606 *data = bap->oa_value; 3607 *numelems = bap->oa_value_len; 3608 3609 return (CRYPTO_SUCCESS); 3610 } 3611 3612 /* 3613 * Finds entry of specified name. If it is not found dca_find_attribute returns 3614 * NULL. 3615 */ 3616 crypto_object_attribute_t * 3617 dca_find_attribute(crypto_object_attribute_t *attrp, uint_t atnum, 3618 uint64_t atype) 3619 { 3620 while (atnum) { 3621 if (attrp->oa_type == atype) 3622 return (attrp); 3623 atnum--; 3624 attrp++; 3625 } 3626 return (NULL); 3627 } 3628 3629 /* 3630 * Return the address of the first data buffer. If the data format is 3631 * unrecognised return NULL. 3632 */ 3633 caddr_t 3634 dca_bufdaddr(crypto_data_t *data) 3635 { 3636 switch (data->cd_format) { 3637 case CRYPTO_DATA_RAW: 3638 return (data->cd_raw.iov_base + data->cd_offset); 3639 case CRYPTO_DATA_UIO: 3640 return (data->cd_uio->uio_iov[0].iov_base + data->cd_offset); 3641 case CRYPTO_DATA_MBLK: 3642 return ((char *)data->cd_mp->b_rptr + data->cd_offset); 3643 default: 3644 DBG(NULL, DWARN, 3645 "dca_bufdaddr: unrecognised crypto data format"); 3646 return (NULL); 3647 } 3648 } 3649 3650 static caddr_t 3651 dca_bufdaddr_out(crypto_data_t *data) 3652 { 3653 size_t offset = data->cd_offset + data->cd_length; 3654 3655 switch (data->cd_format) { 3656 case CRYPTO_DATA_RAW: 3657 return (data->cd_raw.iov_base + offset); 3658 case CRYPTO_DATA_UIO: 3659 return (data->cd_uio->uio_iov[0].iov_base + offset); 3660 case CRYPTO_DATA_MBLK: 3661 return ((char *)data->cd_mp->b_rptr + offset); 3662 default: 3663 DBG(NULL, DWARN, 3664 "dca_bufdaddr_out: unrecognised crypto data format"); 3665 return (NULL); 3666 } 3667 } 3668 3669 /* 3670 * Control entry points. 3671 */ 3672 3673 /* ARGSUSED */ 3674 static void 3675 dca_provider_status(crypto_provider_handle_t provider, uint_t *status) 3676 { 3677 *status = CRYPTO_PROVIDER_READY; 3678 } 3679 3680 /* 3681 * Cipher (encrypt/decrypt) entry points. 3682 */ 3683 3684 /* ARGSUSED */ 3685 static int 3686 dca_encrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, 3687 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, 3688 crypto_req_handle_t req) 3689 { 3690 int error = CRYPTO_FAILED; 3691 dca_t *softc; 3692 3693 softc = DCA_SOFTC_FROM_CTX(ctx); 3694 DBG(softc, DENTRY, "dca_encrypt_init: started"); 3695 3696 /* check mechanism */ 3697 switch (mechanism->cm_type) { 3698 case DES_CBC_MECH_INFO_TYPE: 3699 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP, 3700 DR_ENCRYPT); 3701 break; 3702 case DES3_CBC_MECH_INFO_TYPE: 3703 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP, 3704 DR_ENCRYPT | DR_TRIPLE); 3705 break; 3706 case RSA_PKCS_MECH_INFO_TYPE: 3707 case RSA_X_509_MECH_INFO_TYPE: 3708 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); 3709 break; 3710 default: 3711 cmn_err(CE_WARN, "dca_encrypt_init: unexpected mech type " 3712 "0x%llx\n", (unsigned long long)mechanism->cm_type); 3713 error = CRYPTO_MECHANISM_INVALID; 3714 } 3715 3716 DBG(softc, DENTRY, "dca_encrypt_init: done, err = 0x%x", error); 3717 3718 if (error == CRYPTO_SUCCESS) 3719 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, 3720 &softc->dca_ctx_list_lock); 3721 3722 return (error); 3723 } 3724 3725 /* ARGSUSED */ 3726 static int 3727 dca_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext, 3728 crypto_data_t *ciphertext, crypto_req_handle_t req) 3729 { 3730 int error = CRYPTO_FAILED; 3731 dca_t *softc; 3732 3733 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 3734 return (CRYPTO_OPERATION_NOT_INITIALIZED); 3735 3736 softc = DCA_SOFTC_FROM_CTX(ctx); 3737 DBG(softc, DENTRY, "dca_encrypt: started"); 3738 3739 /* handle inplace ops */ 3740 if (!ciphertext) { 3741 dca_request_t *reqp = ctx->cc_provider_private; 3742 reqp->dr_flags |= DR_INPLACE; 3743 ciphertext = plaintext; 3744 } 3745 3746 /* check mechanism */ 3747 switch (DCA_MECH_FROM_CTX(ctx)) { 3748 case DES_CBC_MECH_INFO_TYPE: 3749 error = dca_3des(ctx, plaintext, ciphertext, req, DR_ENCRYPT); 3750 break; 3751 case DES3_CBC_MECH_INFO_TYPE: 3752 error = dca_3des(ctx, plaintext, ciphertext, req, 3753 DR_ENCRYPT | DR_TRIPLE); 3754 break; 3755 case RSA_PKCS_MECH_INFO_TYPE: 3756 case RSA_X_509_MECH_INFO_TYPE: 3757 error = dca_rsastart(ctx, plaintext, ciphertext, req, 3758 DCA_RSA_ENC); 3759 break; 3760 default: 3761 /* Should never reach here */ 3762 cmn_err(CE_WARN, "dca_encrypt: unexpected mech type " 3763 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 3764 error = CRYPTO_MECHANISM_INVALID; 3765 } 3766 3767 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) && 3768 (error != CRYPTO_BUFFER_TOO_SMALL)) { 3769 ciphertext->cd_length = 0; 3770 } 3771 3772 DBG(softc, DENTRY, "dca_encrypt: done, err = 0x%x", error); 3773 3774 return (error); 3775 } 3776 3777 /* ARGSUSED */ 3778 static int 3779 dca_encrypt_update(crypto_ctx_t *ctx, crypto_data_t *plaintext, 3780 crypto_data_t *ciphertext, crypto_req_handle_t req) 3781 { 3782 int error = CRYPTO_FAILED; 3783 dca_t *softc; 3784 3785 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 3786 return (CRYPTO_OPERATION_NOT_INITIALIZED); 3787 3788 softc = DCA_SOFTC_FROM_CTX(ctx); 3789 DBG(softc, DENTRY, "dca_encrypt_update: started"); 3790 3791 /* handle inplace ops */ 3792 if (!ciphertext) { 3793 dca_request_t *reqp = ctx->cc_provider_private; 3794 reqp->dr_flags |= DR_INPLACE; 3795 ciphertext = plaintext; 3796 } 3797 3798 /* check mechanism */ 3799 switch (DCA_MECH_FROM_CTX(ctx)) { 3800 case DES_CBC_MECH_INFO_TYPE: 3801 error = dca_3desupdate(ctx, plaintext, ciphertext, req, 3802 DR_ENCRYPT); 3803 break; 3804 case DES3_CBC_MECH_INFO_TYPE: 3805 error = dca_3desupdate(ctx, plaintext, ciphertext, req, 3806 DR_ENCRYPT | DR_TRIPLE); 3807 break; 3808 default: 3809 /* Should never reach here */ 3810 cmn_err(CE_WARN, "dca_encrypt_update: unexpected mech type " 3811 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 3812 error = CRYPTO_MECHANISM_INVALID; 3813 } 3814 3815 DBG(softc, DENTRY, "dca_encrypt_update: done, err = 0x%x", error); 3816 3817 return (error); 3818 } 3819 3820 /* ARGSUSED */ 3821 static int 3822 dca_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *ciphertext, 3823 crypto_req_handle_t req) 3824 { 3825 int error = CRYPTO_FAILED; 3826 dca_t *softc; 3827 3828 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 3829 return (CRYPTO_OPERATION_NOT_INITIALIZED); 3830 3831 softc = DCA_SOFTC_FROM_CTX(ctx); 3832 DBG(softc, DENTRY, "dca_encrypt_final: started"); 3833 3834 /* check mechanism */ 3835 switch (DCA_MECH_FROM_CTX(ctx)) { 3836 case DES_CBC_MECH_INFO_TYPE: 3837 error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT); 3838 break; 3839 case DES3_CBC_MECH_INFO_TYPE: 3840 error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT | DR_TRIPLE); 3841 break; 3842 default: 3843 /* Should never reach here */ 3844 cmn_err(CE_WARN, "dca_encrypt_final: unexpected mech type " 3845 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 3846 error = CRYPTO_MECHANISM_INVALID; 3847 } 3848 3849 DBG(softc, DENTRY, "dca_encrypt_final: done, err = 0x%x", error); 3850 3851 return (error); 3852 } 3853 3854 /* ARGSUSED */ 3855 static int 3856 dca_encrypt_atomic(crypto_provider_handle_t provider, 3857 crypto_session_id_t session_id, crypto_mechanism_t *mechanism, 3858 crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext, 3859 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) 3860 { 3861 int error = CRYPTO_FAILED; 3862 dca_t *softc = (dca_t *)provider; 3863 3864 DBG(softc, DENTRY, "dca_encrypt_atomic: started"); 3865 3866 if (ctx_template != NULL) 3867 return (CRYPTO_ARGUMENTS_BAD); 3868 3869 /* handle inplace ops */ 3870 if (!ciphertext) { 3871 ciphertext = plaintext; 3872 } 3873 3874 /* check mechanism */ 3875 switch (mechanism->cm_type) { 3876 case DES_CBC_MECH_INFO_TYPE: 3877 error = dca_3desatomic(provider, session_id, mechanism, key, 3878 plaintext, ciphertext, KM_SLEEP, req, 3879 DR_ENCRYPT | DR_ATOMIC); 3880 break; 3881 case DES3_CBC_MECH_INFO_TYPE: 3882 error = dca_3desatomic(provider, session_id, mechanism, key, 3883 plaintext, ciphertext, KM_SLEEP, req, 3884 DR_ENCRYPT | DR_TRIPLE | DR_ATOMIC); 3885 break; 3886 case RSA_PKCS_MECH_INFO_TYPE: 3887 case RSA_X_509_MECH_INFO_TYPE: 3888 error = dca_rsaatomic(provider, session_id, mechanism, key, 3889 plaintext, ciphertext, KM_SLEEP, req, DCA_RSA_ENC); 3890 break; 3891 default: 3892 cmn_err(CE_WARN, "dca_encrypt_atomic: unexpected mech type " 3893 "0x%llx\n", (unsigned long long)mechanism->cm_type); 3894 error = CRYPTO_MECHANISM_INVALID; 3895 } 3896 3897 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) { 3898 ciphertext->cd_length = 0; 3899 } 3900 3901 DBG(softc, DENTRY, "dca_encrypt_atomic: done, err = 0x%x", error); 3902 3903 return (error); 3904 } 3905 3906 /* ARGSUSED */ 3907 static int 3908 dca_decrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, 3909 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, 3910 crypto_req_handle_t req) 3911 { 3912 int error = CRYPTO_FAILED; 3913 dca_t *softc; 3914 3915 softc = DCA_SOFTC_FROM_CTX(ctx); 3916 DBG(softc, DENTRY, "dca_decrypt_init: started"); 3917 3918 /* check mechanism */ 3919 switch (mechanism->cm_type) { 3920 case DES_CBC_MECH_INFO_TYPE: 3921 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP, 3922 DR_DECRYPT); 3923 break; 3924 case DES3_CBC_MECH_INFO_TYPE: 3925 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP, 3926 DR_DECRYPT | DR_TRIPLE); 3927 break; 3928 case RSA_PKCS_MECH_INFO_TYPE: 3929 case RSA_X_509_MECH_INFO_TYPE: 3930 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); 3931 break; 3932 default: 3933 cmn_err(CE_WARN, "dca_decrypt_init: unexpected mech type " 3934 "0x%llx\n", (unsigned long long)mechanism->cm_type); 3935 error = CRYPTO_MECHANISM_INVALID; 3936 } 3937 3938 DBG(softc, DENTRY, "dca_decrypt_init: done, err = 0x%x", error); 3939 3940 if (error == CRYPTO_SUCCESS) 3941 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, 3942 &softc->dca_ctx_list_lock); 3943 3944 return (error); 3945 } 3946 3947 /* ARGSUSED */ 3948 static int 3949 dca_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext, 3950 crypto_data_t *plaintext, crypto_req_handle_t req) 3951 { 3952 int error = CRYPTO_FAILED; 3953 dca_t *softc; 3954 3955 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 3956 return (CRYPTO_OPERATION_NOT_INITIALIZED); 3957 3958 softc = DCA_SOFTC_FROM_CTX(ctx); 3959 DBG(softc, DENTRY, "dca_decrypt: started"); 3960 3961 /* handle inplace ops */ 3962 if (!plaintext) { 3963 dca_request_t *reqp = ctx->cc_provider_private; 3964 reqp->dr_flags |= DR_INPLACE; 3965 plaintext = ciphertext; 3966 } 3967 3968 /* check mechanism */ 3969 switch (DCA_MECH_FROM_CTX(ctx)) { 3970 case DES_CBC_MECH_INFO_TYPE: 3971 error = dca_3des(ctx, ciphertext, plaintext, req, DR_DECRYPT); 3972 break; 3973 case DES3_CBC_MECH_INFO_TYPE: 3974 error = dca_3des(ctx, ciphertext, plaintext, req, 3975 DR_DECRYPT | DR_TRIPLE); 3976 break; 3977 case RSA_PKCS_MECH_INFO_TYPE: 3978 case RSA_X_509_MECH_INFO_TYPE: 3979 error = dca_rsastart(ctx, ciphertext, plaintext, req, 3980 DCA_RSA_DEC); 3981 break; 3982 default: 3983 /* Should never reach here */ 3984 cmn_err(CE_WARN, "dca_decrypt: unexpected mech type " 3985 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 3986 error = CRYPTO_MECHANISM_INVALID; 3987 } 3988 3989 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) && 3990 (error != CRYPTO_BUFFER_TOO_SMALL)) { 3991 if (plaintext) 3992 plaintext->cd_length = 0; 3993 } 3994 3995 DBG(softc, DENTRY, "dca_decrypt: done, err = 0x%x", error); 3996 3997 return (error); 3998 } 3999 4000 /* ARGSUSED */ 4001 static int 4002 dca_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext, 4003 crypto_data_t *plaintext, crypto_req_handle_t req) 4004 { 4005 int error = CRYPTO_FAILED; 4006 dca_t *softc; 4007 4008 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4009 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4010 4011 softc = DCA_SOFTC_FROM_CTX(ctx); 4012 DBG(softc, DENTRY, "dca_decrypt_update: started"); 4013 4014 /* handle inplace ops */ 4015 if (!plaintext) { 4016 dca_request_t *reqp = ctx->cc_provider_private; 4017 reqp->dr_flags |= DR_INPLACE; 4018 plaintext = ciphertext; 4019 } 4020 4021 /* check mechanism */ 4022 switch (DCA_MECH_FROM_CTX(ctx)) { 4023 case DES_CBC_MECH_INFO_TYPE: 4024 error = dca_3desupdate(ctx, ciphertext, plaintext, req, 4025 DR_DECRYPT); 4026 break; 4027 case DES3_CBC_MECH_INFO_TYPE: 4028 error = dca_3desupdate(ctx, ciphertext, plaintext, req, 4029 DR_DECRYPT | DR_TRIPLE); 4030 break; 4031 default: 4032 /* Should never reach here */ 4033 cmn_err(CE_WARN, "dca_decrypt_update: unexpected mech type " 4034 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4035 error = CRYPTO_MECHANISM_INVALID; 4036 } 4037 4038 DBG(softc, DENTRY, "dca_decrypt_update: done, err = 0x%x", error); 4039 4040 return (error); 4041 } 4042 4043 /* ARGSUSED */ 4044 static int 4045 dca_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *plaintext, 4046 crypto_req_handle_t req) 4047 { 4048 int error = CRYPTO_FAILED; 4049 dca_t *softc; 4050 4051 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4052 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4053 4054 softc = DCA_SOFTC_FROM_CTX(ctx); 4055 DBG(softc, DENTRY, "dca_decrypt_final: started"); 4056 4057 /* check mechanism */ 4058 switch (DCA_MECH_FROM_CTX(ctx)) { 4059 case DES_CBC_MECH_INFO_TYPE: 4060 error = dca_3desfinal(ctx, plaintext, DR_DECRYPT); 4061 break; 4062 case DES3_CBC_MECH_INFO_TYPE: 4063 error = dca_3desfinal(ctx, plaintext, DR_DECRYPT | DR_TRIPLE); 4064 break; 4065 default: 4066 /* Should never reach here */ 4067 cmn_err(CE_WARN, "dca_decrypt_final: unexpected mech type " 4068 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4069 error = CRYPTO_MECHANISM_INVALID; 4070 } 4071 4072 DBG(softc, DENTRY, "dca_decrypt_final: done, err = 0x%x", error); 4073 4074 return (error); 4075 } 4076 4077 /* ARGSUSED */ 4078 static int 4079 dca_decrypt_atomic(crypto_provider_handle_t provider, 4080 crypto_session_id_t session_id, crypto_mechanism_t *mechanism, 4081 crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext, 4082 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) 4083 { 4084 int error = CRYPTO_FAILED; 4085 dca_t *softc = (dca_t *)provider; 4086 4087 DBG(softc, DENTRY, "dca_decrypt_atomic: started"); 4088 4089 if (ctx_template != NULL) 4090 return (CRYPTO_ARGUMENTS_BAD); 4091 4092 /* handle inplace ops */ 4093 if (!plaintext) { 4094 plaintext = ciphertext; 4095 } 4096 4097 /* check mechanism */ 4098 switch (mechanism->cm_type) { 4099 case DES_CBC_MECH_INFO_TYPE: 4100 error = dca_3desatomic(provider, session_id, mechanism, key, 4101 ciphertext, plaintext, KM_SLEEP, req, 4102 DR_DECRYPT | DR_ATOMIC); 4103 break; 4104 case DES3_CBC_MECH_INFO_TYPE: 4105 error = dca_3desatomic(provider, session_id, mechanism, key, 4106 ciphertext, plaintext, KM_SLEEP, req, 4107 DR_DECRYPT | DR_TRIPLE | DR_ATOMIC); 4108 break; 4109 case RSA_PKCS_MECH_INFO_TYPE: 4110 case RSA_X_509_MECH_INFO_TYPE: 4111 error = dca_rsaatomic(provider, session_id, mechanism, key, 4112 ciphertext, plaintext, KM_SLEEP, req, DCA_RSA_DEC); 4113 break; 4114 default: 4115 cmn_err(CE_WARN, "dca_decrypt_atomic: unexpected mech type " 4116 "0x%llx\n", (unsigned long long)mechanism->cm_type); 4117 error = CRYPTO_MECHANISM_INVALID; 4118 } 4119 4120 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) { 4121 plaintext->cd_length = 0; 4122 } 4123 4124 DBG(softc, DENTRY, "dca_decrypt_atomic: done, err = 0x%x", error); 4125 4126 return (error); 4127 } 4128 4129 /* 4130 * Sign entry points. 4131 */ 4132 4133 /* ARGSUSED */ 4134 static int 4135 dca_sign_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, 4136 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, 4137 crypto_req_handle_t req) 4138 { 4139 int error = CRYPTO_FAILED; 4140 dca_t *softc; 4141 4142 softc = DCA_SOFTC_FROM_CTX(ctx); 4143 DBG(softc, DENTRY, "dca_sign_init: started\n"); 4144 4145 if (ctx_template != NULL) 4146 return (CRYPTO_ARGUMENTS_BAD); 4147 4148 /* check mechanism */ 4149 switch (mechanism->cm_type) { 4150 case RSA_PKCS_MECH_INFO_TYPE: 4151 case RSA_X_509_MECH_INFO_TYPE: 4152 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); 4153 break; 4154 case DSA_MECH_INFO_TYPE: 4155 error = dca_dsainit(ctx, mechanism, key, KM_SLEEP, 4156 DCA_DSA_SIGN); 4157 break; 4158 default: 4159 cmn_err(CE_WARN, "dca_sign_init: unexpected mech type " 4160 "0x%llx\n", (unsigned long long)mechanism->cm_type); 4161 error = CRYPTO_MECHANISM_INVALID; 4162 } 4163 4164 DBG(softc, DENTRY, "dca_sign_init: done, err = 0x%x", error); 4165 4166 if (error == CRYPTO_SUCCESS) 4167 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, 4168 &softc->dca_ctx_list_lock); 4169 4170 return (error); 4171 } 4172 4173 static int 4174 dca_sign(crypto_ctx_t *ctx, crypto_data_t *data, 4175 crypto_data_t *signature, crypto_req_handle_t req) 4176 { 4177 int error = CRYPTO_FAILED; 4178 dca_t *softc; 4179 4180 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4181 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4182 4183 softc = DCA_SOFTC_FROM_CTX(ctx); 4184 DBG(softc, DENTRY, "dca_sign: started\n"); 4185 4186 /* check mechanism */ 4187 switch (DCA_MECH_FROM_CTX(ctx)) { 4188 case RSA_PKCS_MECH_INFO_TYPE: 4189 case RSA_X_509_MECH_INFO_TYPE: 4190 error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGN); 4191 break; 4192 case DSA_MECH_INFO_TYPE: 4193 error = dca_dsa_sign(ctx, data, signature, req); 4194 break; 4195 default: 4196 cmn_err(CE_WARN, "dca_sign: unexpected mech type " 4197 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4198 error = CRYPTO_MECHANISM_INVALID; 4199 } 4200 4201 DBG(softc, DENTRY, "dca_sign: done, err = 0x%x", error); 4202 4203 return (error); 4204 } 4205 4206 /* ARGSUSED */ 4207 static int 4208 dca_sign_update(crypto_ctx_t *ctx, crypto_data_t *data, 4209 crypto_req_handle_t req) 4210 { 4211 int error = CRYPTO_MECHANISM_INVALID; 4212 dca_t *softc; 4213 4214 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4215 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4216 4217 softc = DCA_SOFTC_FROM_CTX(ctx); 4218 DBG(softc, DENTRY, "dca_sign_update: started\n"); 4219 4220 cmn_err(CE_WARN, "dca_sign_update: unexpected mech type " 4221 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4222 4223 DBG(softc, DENTRY, "dca_sign_update: done, err = 0x%x", error); 4224 4225 return (error); 4226 } 4227 4228 /* ARGSUSED */ 4229 static int 4230 dca_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature, 4231 crypto_req_handle_t req) 4232 { 4233 int error = CRYPTO_MECHANISM_INVALID; 4234 dca_t *softc; 4235 4236 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4237 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4238 4239 softc = DCA_SOFTC_FROM_CTX(ctx); 4240 DBG(softc, DENTRY, "dca_sign_final: started\n"); 4241 4242 cmn_err(CE_WARN, "dca_sign_final: unexpected mech type " 4243 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4244 4245 DBG(softc, DENTRY, "dca_sign_final: done, err = 0x%x", error); 4246 4247 return (error); 4248 } 4249 4250 static int 4251 dca_sign_atomic(crypto_provider_handle_t provider, 4252 crypto_session_id_t session_id, crypto_mechanism_t *mechanism, 4253 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature, 4254 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) 4255 { 4256 int error = CRYPTO_FAILED; 4257 dca_t *softc = (dca_t *)provider; 4258 4259 DBG(softc, DENTRY, "dca_sign_atomic: started\n"); 4260 4261 if (ctx_template != NULL) 4262 return (CRYPTO_ARGUMENTS_BAD); 4263 4264 /* check mechanism */ 4265 switch (mechanism->cm_type) { 4266 case RSA_PKCS_MECH_INFO_TYPE: 4267 case RSA_X_509_MECH_INFO_TYPE: 4268 error = dca_rsaatomic(provider, session_id, mechanism, key, 4269 data, signature, KM_SLEEP, req, DCA_RSA_SIGN); 4270 break; 4271 case DSA_MECH_INFO_TYPE: 4272 error = dca_dsaatomic(provider, session_id, mechanism, key, 4273 data, signature, KM_SLEEP, req, DCA_DSA_SIGN); 4274 break; 4275 default: 4276 cmn_err(CE_WARN, "dca_sign_atomic: unexpected mech type " 4277 "0x%llx\n", (unsigned long long)mechanism->cm_type); 4278 error = CRYPTO_MECHANISM_INVALID; 4279 } 4280 4281 DBG(softc, DENTRY, "dca_sign_atomic: done, err = 0x%x", error); 4282 4283 return (error); 4284 } 4285 4286 /* ARGSUSED */ 4287 static int 4288 dca_sign_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, 4289 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, 4290 crypto_req_handle_t req) 4291 { 4292 int error = CRYPTO_FAILED; 4293 dca_t *softc; 4294 4295 softc = DCA_SOFTC_FROM_CTX(ctx); 4296 DBG(softc, DENTRY, "dca_sign_recover_init: started\n"); 4297 4298 if (ctx_template != NULL) 4299 return (CRYPTO_ARGUMENTS_BAD); 4300 4301 /* check mechanism */ 4302 switch (mechanism->cm_type) { 4303 case RSA_PKCS_MECH_INFO_TYPE: 4304 case RSA_X_509_MECH_INFO_TYPE: 4305 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); 4306 break; 4307 default: 4308 cmn_err(CE_WARN, "dca_sign_recover_init: unexpected mech type " 4309 "0x%llx\n", (unsigned long long)mechanism->cm_type); 4310 error = CRYPTO_MECHANISM_INVALID; 4311 } 4312 4313 DBG(softc, DENTRY, "dca_sign_recover_init: done, err = 0x%x", error); 4314 4315 if (error == CRYPTO_SUCCESS) 4316 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, 4317 &softc->dca_ctx_list_lock); 4318 4319 return (error); 4320 } 4321 4322 static int 4323 dca_sign_recover(crypto_ctx_t *ctx, crypto_data_t *data, 4324 crypto_data_t *signature, crypto_req_handle_t req) 4325 { 4326 int error = CRYPTO_FAILED; 4327 dca_t *softc; 4328 4329 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4330 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4331 4332 softc = DCA_SOFTC_FROM_CTX(ctx); 4333 DBG(softc, DENTRY, "dca_sign_recover: started\n"); 4334 4335 /* check mechanism */ 4336 switch (DCA_MECH_FROM_CTX(ctx)) { 4337 case RSA_PKCS_MECH_INFO_TYPE: 4338 case RSA_X_509_MECH_INFO_TYPE: 4339 error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGNR); 4340 break; 4341 default: 4342 cmn_err(CE_WARN, "dca_sign_recover: unexpected mech type " 4343 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4344 error = CRYPTO_MECHANISM_INVALID; 4345 } 4346 4347 DBG(softc, DENTRY, "dca_sign_recover: done, err = 0x%x", error); 4348 4349 return (error); 4350 } 4351 4352 static int 4353 dca_sign_recover_atomic(crypto_provider_handle_t provider, 4354 crypto_session_id_t session_id, crypto_mechanism_t *mechanism, 4355 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature, 4356 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) 4357 { 4358 int error = CRYPTO_FAILED; 4359 dca_t *softc = (dca_t *)provider; 4360 4361 DBG(softc, DENTRY, "dca_sign_recover_atomic: started\n"); 4362 4363 if (ctx_template != NULL) 4364 return (CRYPTO_ARGUMENTS_BAD); 4365 4366 /* check mechanism */ 4367 switch (mechanism->cm_type) { 4368 case RSA_PKCS_MECH_INFO_TYPE: 4369 case RSA_X_509_MECH_INFO_TYPE: 4370 error = dca_rsaatomic(provider, session_id, mechanism, key, 4371 data, signature, KM_SLEEP, req, DCA_RSA_SIGNR); 4372 break; 4373 default: 4374 cmn_err(CE_WARN, "dca_sign_recover_atomic: unexpected mech type" 4375 " 0x%llx\n", (unsigned long long)mechanism->cm_type); 4376 error = CRYPTO_MECHANISM_INVALID; 4377 } 4378 4379 DBG(softc, DENTRY, "dca_sign_recover_atomic: done, err = 0x%x", error); 4380 4381 return (error); 4382 } 4383 4384 /* 4385 * Verify entry points. 4386 */ 4387 4388 /* ARGSUSED */ 4389 static int 4390 dca_verify_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, 4391 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, 4392 crypto_req_handle_t req) 4393 { 4394 int error = CRYPTO_FAILED; 4395 dca_t *softc; 4396 4397 softc = DCA_SOFTC_FROM_CTX(ctx); 4398 DBG(softc, DENTRY, "dca_verify_init: started\n"); 4399 4400 if (ctx_template != NULL) 4401 return (CRYPTO_ARGUMENTS_BAD); 4402 4403 /* check mechanism */ 4404 switch (mechanism->cm_type) { 4405 case RSA_PKCS_MECH_INFO_TYPE: 4406 case RSA_X_509_MECH_INFO_TYPE: 4407 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); 4408 break; 4409 case DSA_MECH_INFO_TYPE: 4410 error = dca_dsainit(ctx, mechanism, key, KM_SLEEP, 4411 DCA_DSA_VRFY); 4412 break; 4413 default: 4414 cmn_err(CE_WARN, "dca_verify_init: unexpected mech type " 4415 "0x%llx\n", (unsigned long long)mechanism->cm_type); 4416 error = CRYPTO_MECHANISM_INVALID; 4417 } 4418 4419 DBG(softc, DENTRY, "dca_verify_init: done, err = 0x%x", error); 4420 4421 if (error == CRYPTO_SUCCESS) 4422 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, 4423 &softc->dca_ctx_list_lock); 4424 4425 return (error); 4426 } 4427 4428 static int 4429 dca_verify(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature, 4430 crypto_req_handle_t req) 4431 { 4432 int error = CRYPTO_FAILED; 4433 dca_t *softc; 4434 4435 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4436 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4437 4438 softc = DCA_SOFTC_FROM_CTX(ctx); 4439 DBG(softc, DENTRY, "dca_verify: started\n"); 4440 4441 /* check mechanism */ 4442 switch (DCA_MECH_FROM_CTX(ctx)) { 4443 case RSA_PKCS_MECH_INFO_TYPE: 4444 case RSA_X_509_MECH_INFO_TYPE: 4445 error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFY); 4446 break; 4447 case DSA_MECH_INFO_TYPE: 4448 error = dca_dsa_verify(ctx, data, signature, req); 4449 break; 4450 default: 4451 cmn_err(CE_WARN, "dca_verify: unexpected mech type " 4452 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4453 error = CRYPTO_MECHANISM_INVALID; 4454 } 4455 4456 DBG(softc, DENTRY, "dca_verify: done, err = 0x%x", error); 4457 4458 return (error); 4459 } 4460 4461 /* ARGSUSED */ 4462 static int 4463 dca_verify_update(crypto_ctx_t *ctx, crypto_data_t *data, 4464 crypto_req_handle_t req) 4465 { 4466 int error = CRYPTO_MECHANISM_INVALID; 4467 dca_t *softc; 4468 4469 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4470 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4471 4472 softc = DCA_SOFTC_FROM_CTX(ctx); 4473 DBG(softc, DENTRY, "dca_verify_update: started\n"); 4474 4475 cmn_err(CE_WARN, "dca_verify_update: unexpected mech type " 4476 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4477 4478 DBG(softc, DENTRY, "dca_verify_update: done, err = 0x%x", error); 4479 4480 return (error); 4481 } 4482 4483 /* ARGSUSED */ 4484 static int 4485 dca_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature, 4486 crypto_req_handle_t req) 4487 { 4488 int error = CRYPTO_MECHANISM_INVALID; 4489 dca_t *softc; 4490 4491 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4492 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4493 4494 softc = DCA_SOFTC_FROM_CTX(ctx); 4495 DBG(softc, DENTRY, "dca_verify_final: started\n"); 4496 4497 cmn_err(CE_WARN, "dca_verify_final: unexpected mech type " 4498 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4499 4500 DBG(softc, DENTRY, "dca_verify_final: done, err = 0x%x", error); 4501 4502 return (error); 4503 } 4504 4505 static int 4506 dca_verify_atomic(crypto_provider_handle_t provider, 4507 crypto_session_id_t session_id, crypto_mechanism_t *mechanism, 4508 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature, 4509 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) 4510 { 4511 int error = CRYPTO_FAILED; 4512 dca_t *softc = (dca_t *)provider; 4513 4514 DBG(softc, DENTRY, "dca_verify_atomic: started\n"); 4515 4516 if (ctx_template != NULL) 4517 return (CRYPTO_ARGUMENTS_BAD); 4518 4519 /* check mechanism */ 4520 switch (mechanism->cm_type) { 4521 case RSA_PKCS_MECH_INFO_TYPE: 4522 case RSA_X_509_MECH_INFO_TYPE: 4523 error = dca_rsaatomic(provider, session_id, mechanism, key, 4524 signature, data, KM_SLEEP, req, DCA_RSA_VRFY); 4525 break; 4526 case DSA_MECH_INFO_TYPE: 4527 error = dca_dsaatomic(provider, session_id, mechanism, key, 4528 data, signature, KM_SLEEP, req, DCA_DSA_VRFY); 4529 break; 4530 default: 4531 cmn_err(CE_WARN, "dca_verify_atomic: unexpected mech type " 4532 "0x%llx\n", (unsigned long long)mechanism->cm_type); 4533 error = CRYPTO_MECHANISM_INVALID; 4534 } 4535 4536 DBG(softc, DENTRY, "dca_verify_atomic: done, err = 0x%x", error); 4537 4538 return (error); 4539 } 4540 4541 /* ARGSUSED */ 4542 static int 4543 dca_verify_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, 4544 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, 4545 crypto_req_handle_t req) 4546 { 4547 int error = CRYPTO_MECHANISM_INVALID; 4548 dca_t *softc; 4549 4550 softc = DCA_SOFTC_FROM_CTX(ctx); 4551 DBG(softc, DENTRY, "dca_verify_recover_init: started\n"); 4552 4553 if (ctx_template != NULL) 4554 return (CRYPTO_ARGUMENTS_BAD); 4555 4556 /* check mechanism */ 4557 switch (mechanism->cm_type) { 4558 case RSA_PKCS_MECH_INFO_TYPE: 4559 case RSA_X_509_MECH_INFO_TYPE: 4560 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); 4561 break; 4562 default: 4563 cmn_err(CE_WARN, "dca_verify_recover_init: unexpected mech type" 4564 " 0x%llx\n", (unsigned long long)mechanism->cm_type); 4565 } 4566 4567 DBG(softc, DENTRY, "dca_verify_recover_init: done, err = 0x%x", error); 4568 4569 if (error == CRYPTO_SUCCESS) 4570 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, 4571 &softc->dca_ctx_list_lock); 4572 4573 return (error); 4574 } 4575 4576 static int 4577 dca_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature, 4578 crypto_data_t *data, crypto_req_handle_t req) 4579 { 4580 int error = CRYPTO_MECHANISM_INVALID; 4581 dca_t *softc; 4582 4583 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4584 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4585 4586 softc = DCA_SOFTC_FROM_CTX(ctx); 4587 DBG(softc, DENTRY, "dca_verify_recover: started\n"); 4588 4589 /* check mechanism */ 4590 switch (DCA_MECH_FROM_CTX(ctx)) { 4591 case RSA_PKCS_MECH_INFO_TYPE: 4592 case RSA_X_509_MECH_INFO_TYPE: 4593 error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFYR); 4594 break; 4595 default: 4596 cmn_err(CE_WARN, "dca_verify_recover: unexpected mech type " 4597 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4598 } 4599 4600 DBG(softc, DENTRY, "dca_verify_recover: done, err = 0x%x", error); 4601 4602 return (error); 4603 } 4604 4605 static int 4606 dca_verify_recover_atomic(crypto_provider_handle_t provider, 4607 crypto_session_id_t session_id, crypto_mechanism_t *mechanism, 4608 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature, 4609 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) 4610 { 4611 int error = CRYPTO_MECHANISM_INVALID; 4612 dca_t *softc = (dca_t *)provider; 4613 4614 DBG(softc, DENTRY, "dca_verify_recover_atomic: started\n"); 4615 4616 if (ctx_template != NULL) 4617 return (CRYPTO_ARGUMENTS_BAD); 4618 4619 /* check mechanism */ 4620 switch (mechanism->cm_type) { 4621 case RSA_PKCS_MECH_INFO_TYPE: 4622 case RSA_X_509_MECH_INFO_TYPE: 4623 error = dca_rsaatomic(provider, session_id, mechanism, key, 4624 signature, data, KM_SLEEP, req, DCA_RSA_VRFYR); 4625 break; 4626 default: 4627 cmn_err(CE_WARN, "dca_verify_recover_atomic: unexpected mech " 4628 "type 0x%llx\n", (unsigned long long)mechanism->cm_type); 4629 error = CRYPTO_MECHANISM_INVALID; 4630 } 4631 4632 DBG(softc, DENTRY, 4633 "dca_verify_recover_atomic: done, err = 0x%x", error); 4634 4635 return (error); 4636 } 4637 4638 /* 4639 * Random number entry points. 4640 */ 4641 4642 /* ARGSUSED */ 4643 static int 4644 dca_generate_random(crypto_provider_handle_t provider, 4645 crypto_session_id_t session_id, 4646 uchar_t *buf, size_t len, crypto_req_handle_t req) 4647 { 4648 int error = CRYPTO_FAILED; 4649 dca_t *softc = (dca_t *)provider; 4650 4651 DBG(softc, DENTRY, "dca_generate_random: started"); 4652 4653 error = dca_rng(softc, buf, len, req); 4654 4655 DBG(softc, DENTRY, "dca_generate_random: done, err = 0x%x", error); 4656 4657 return (error); 4658 } 4659 4660 /* 4661 * Context management entry points. 4662 */ 4663 4664 int 4665 dca_free_context(crypto_ctx_t *ctx) 4666 { 4667 int error = CRYPTO_SUCCESS; 4668 dca_t *softc; 4669 4670 softc = DCA_SOFTC_FROM_CTX(ctx); 4671 DBG(softc, DENTRY, "dca_free_context: entered"); 4672 4673 if (ctx->cc_provider_private == NULL) 4674 return (error); 4675 4676 dca_rmlist2(ctx->cc_provider_private, &softc->dca_ctx_list_lock); 4677 4678 error = dca_free_context_low(ctx); 4679 4680 DBG(softc, DENTRY, "dca_free_context: done, err = 0x%x", error); 4681 4682 return (error); 4683 } 4684 4685 static int 4686 dca_free_context_low(crypto_ctx_t *ctx) 4687 { 4688 int error = CRYPTO_SUCCESS; 4689 4690 /* check mechanism */ 4691 switch (DCA_MECH_FROM_CTX(ctx)) { 4692 case DES_CBC_MECH_INFO_TYPE: 4693 case DES3_CBC_MECH_INFO_TYPE: 4694 dca_3desctxfree(ctx); 4695 break; 4696 case RSA_PKCS_MECH_INFO_TYPE: 4697 case RSA_X_509_MECH_INFO_TYPE: 4698 dca_rsactxfree(ctx); 4699 break; 4700 case DSA_MECH_INFO_TYPE: 4701 dca_dsactxfree(ctx); 4702 break; 4703 default: 4704 /* Should never reach here */ 4705 cmn_err(CE_WARN, "dca_free_context_low: unexpected mech type " 4706 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4707 error = CRYPTO_MECHANISM_INVALID; 4708 } 4709 4710 return (error); 4711 } 4712 4713 4714 /* Free any unfreed private context. It is called in detach. */ 4715 static void 4716 dca_free_context_list(dca_t *dca) 4717 { 4718 dca_listnode_t *node; 4719 crypto_ctx_t ctx; 4720 4721 (void) memset(&ctx, 0, sizeof (ctx)); 4722 ctx.cc_provider = dca; 4723 4724 while ((node = dca_delist2(&dca->dca_ctx_list, 4725 &dca->dca_ctx_list_lock)) != NULL) { 4726 ctx.cc_provider_private = node; 4727 (void) dca_free_context_low(&ctx); 4728 } 4729 } 4730 4731 static int 4732 ext_info_sym(crypto_provider_handle_t prov, 4733 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq) 4734 { 4735 return (ext_info_base(prov, ext_info, cfreq, IDENT_SYM)); 4736 } 4737 4738 static int 4739 ext_info_asym(crypto_provider_handle_t prov, 4740 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq) 4741 { 4742 int rv; 4743 4744 rv = ext_info_base(prov, ext_info, cfreq, IDENT_ASYM); 4745 /* The asymmetric cipher slot supports random */ 4746 ext_info->ei_flags |= CRYPTO_EXTF_RNG; 4747 4748 return (rv); 4749 } 4750 4751 /* ARGSUSED */ 4752 static int 4753 ext_info_base(crypto_provider_handle_t prov, 4754 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id) 4755 { 4756 dca_t *dca = (dca_t *)prov; 4757 int len; 4758 4759 /* Label */ 4760 (void) sprintf((char *)ext_info->ei_label, "%s/%d %s", 4761 ddi_driver_name(dca->dca_dip), ddi_get_instance(dca->dca_dip), id); 4762 len = strlen((char *)ext_info->ei_label); 4763 (void) memset(ext_info->ei_label + len, ' ', 4764 CRYPTO_EXT_SIZE_LABEL - len); 4765 4766 /* Manufacturer ID */ 4767 (void) sprintf((char *)ext_info->ei_manufacturerID, "%s", 4768 DCA_MANUFACTURER_ID); 4769 len = strlen((char *)ext_info->ei_manufacturerID); 4770 (void) memset(ext_info->ei_manufacturerID + len, ' ', 4771 CRYPTO_EXT_SIZE_MANUF - len); 4772 4773 /* Model */ 4774 (void) sprintf((char *)ext_info->ei_model, dca->dca_model); 4775 4776 DBG(dca, DWARN, "kCF MODEL: %s", (char *)ext_info->ei_model); 4777 4778 len = strlen((char *)ext_info->ei_model); 4779 (void) memset(ext_info->ei_model + len, ' ', 4780 CRYPTO_EXT_SIZE_MODEL - len); 4781 4782 /* Serial Number. Blank for Deimos */ 4783 (void) memset(ext_info->ei_serial_number, ' ', CRYPTO_EXT_SIZE_SERIAL); 4784 4785 ext_info->ei_flags = CRYPTO_EXTF_WRITE_PROTECTED; 4786 4787 ext_info->ei_max_session_count = CRYPTO_UNAVAILABLE_INFO; 4788 ext_info->ei_max_pin_len = CRYPTO_UNAVAILABLE_INFO; 4789 ext_info->ei_min_pin_len = CRYPTO_UNAVAILABLE_INFO; 4790 ext_info->ei_total_public_memory = CRYPTO_UNAVAILABLE_INFO; 4791 ext_info->ei_free_public_memory = CRYPTO_UNAVAILABLE_INFO; 4792 ext_info->ei_total_private_memory = CRYPTO_UNAVAILABLE_INFO; 4793 ext_info->ei_free_private_memory = CRYPTO_UNAVAILABLE_INFO; 4794 ext_info->ei_hardware_version.cv_major = 0; 4795 ext_info->ei_hardware_version.cv_minor = 0; 4796 ext_info->ei_firmware_version.cv_major = 0; 4797 ext_info->ei_firmware_version.cv_minor = 0; 4798 4799 /* Time. No need to be supplied for token without a clock */ 4800 ext_info->ei_time[0] = '\000'; 4801 4802 return (CRYPTO_SUCCESS); 4803 } 4804 4805 static void 4806 dca_fma_init(dca_t *dca) 4807 { 4808 ddi_iblock_cookie_t fm_ibc; 4809 int fm_capabilities = DDI_FM_EREPORT_CAPABLE | 4810 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE | 4811 DDI_FM_ERRCB_CAPABLE; 4812 4813 /* Read FMA capabilities from dca.conf file (if present) */ 4814 dca->fm_capabilities = ddi_getprop(DDI_DEV_T_ANY, dca->dca_dip, 4815 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", 4816 fm_capabilities); 4817 4818 DBG(dca, DWARN, "dca->fm_capabilities = 0x%x", dca->fm_capabilities); 4819 4820 /* Only register with IO Fault Services if we have some capability */ 4821 if (dca->fm_capabilities) { 4822 dca_regsattr.devacc_attr_access = DDI_FLAGERR_ACC; 4823 dca_dmaattr.dma_attr_flags = DDI_DMA_FLAGERR; 4824 4825 /* Register capabilities with IO Fault Services */ 4826 ddi_fm_init(dca->dca_dip, &dca->fm_capabilities, &fm_ibc); 4827 DBG(dca, DWARN, "fm_capable() = 0x%x", 4828 ddi_fm_capable(dca->dca_dip)); 4829 4830 /* 4831 * Initialize pci ereport capabilities if ereport capable 4832 */ 4833 if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) || 4834 DDI_FM_ERRCB_CAP(dca->fm_capabilities)) 4835 pci_ereport_setup(dca->dca_dip); 4836 4837 /* 4838 * Initialize callback mutex and register error callback if 4839 * error callback capable. 4840 */ 4841 if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) { 4842 ddi_fm_handler_register(dca->dca_dip, dca_fm_error_cb, 4843 (void *)dca); 4844 } 4845 } else { 4846 /* 4847 * These fields have to be cleared of FMA if there are no 4848 * FMA capabilities at runtime. 4849 */ 4850 dca_regsattr.devacc_attr_access = DDI_DEFAULT_ACC; 4851 dca_dmaattr.dma_attr_flags = 0; 4852 } 4853 } 4854 4855 4856 static void 4857 dca_fma_fini(dca_t *dca) 4858 { 4859 /* Only unregister FMA capabilities if we registered some */ 4860 if (dca->fm_capabilities) { 4861 4862 /* 4863 * Release any resources allocated by pci_ereport_setup() 4864 */ 4865 if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) || 4866 DDI_FM_ERRCB_CAP(dca->fm_capabilities)) { 4867 pci_ereport_teardown(dca->dca_dip); 4868 } 4869 4870 /* 4871 * Free callback mutex and un-register error callback if 4872 * error callback capable. 4873 */ 4874 if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) { 4875 ddi_fm_handler_unregister(dca->dca_dip); 4876 } 4877 4878 /* Unregister from IO Fault Services */ 4879 ddi_fm_fini(dca->dca_dip); 4880 DBG(dca, DWARN, "fm_capable() = 0x%x", 4881 ddi_fm_capable(dca->dca_dip)); 4882 } 4883 } 4884 4885 4886 /* 4887 * The IO fault service error handling callback function 4888 */ 4889 /*ARGSUSED*/ 4890 static int 4891 dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 4892 { 4893 dca_t *dca = (dca_t *)impl_data; 4894 4895 pci_ereport_post(dip, err, NULL); 4896 if (err->fme_status == DDI_FM_FATAL) { 4897 dca_failure(dca, DDI_DATAPATH_FAULT, 4898 DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR, 4899 "fault PCI in FMA callback."); 4900 } 4901 return (err->fme_status); 4902 } 4903 4904 4905 static int 4906 dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle, 4907 dca_fma_eclass_t eclass_index) 4908 { 4909 ddi_fm_error_t de; 4910 int version = 0; 4911 4912 ddi_fm_acc_err_get(handle, &de, version); 4913 if (de.fme_status != DDI_FM_OK) { 4914 dca_failure(dca, DDI_DATAPATH_FAULT, 4915 eclass_index, fm_ena_increment(de.fme_ena), 4916 CRYPTO_DEVICE_ERROR, ""); 4917 return (DDI_FAILURE); 4918 } 4919 4920 return (DDI_SUCCESS); 4921 } 4922 4923 int 4924 dca_check_dma_handle(dca_t *dca, ddi_dma_handle_t handle, 4925 dca_fma_eclass_t eclass_index) 4926 { 4927 ddi_fm_error_t de; 4928 int version = 0; 4929 4930 ddi_fm_dma_err_get(handle, &de, version); 4931 if (de.fme_status != DDI_FM_OK) { 4932 dca_failure(dca, DDI_DATAPATH_FAULT, 4933 eclass_index, fm_ena_increment(de.fme_ena), 4934 CRYPTO_DEVICE_ERROR, ""); 4935 return (DDI_FAILURE); 4936 } 4937 return (DDI_SUCCESS); 4938 } 4939 4940 static uint64_t 4941 dca_ena(uint64_t ena) 4942 { 4943 if (ena == 0) 4944 ena = fm_ena_generate(0, FM_ENA_FMT1); 4945 else 4946 ena = fm_ena_increment(ena); 4947 return (ena); 4948 } 4949 4950 static char * 4951 dca_fma_eclass_string(char *model, dca_fma_eclass_t index) 4952 { 4953 if (strstr(model, "500")) 4954 return (dca_fma_eclass_sca500[index]); 4955 else 4956 return (dca_fma_eclass_sca1000[index]); 4957 } 4958