1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 28 /* 29 * Deimos - cryptographic acceleration based upon Broadcom 582x. 30 */ 31 32 #include <sys/types.h> 33 #include <sys/modctl.h> 34 #include <sys/conf.h> 35 #include <sys/devops.h> 36 #include <sys/ddi.h> 37 #include <sys/sunddi.h> 38 #include <sys/cmn_err.h> 39 #include <sys/varargs.h> 40 #include <sys/file.h> 41 #include <sys/stat.h> 42 #include <sys/kmem.h> 43 #include <sys/ioccom.h> 44 #include <sys/open.h> 45 #include <sys/cred.h> 46 #include <sys/kstat.h> 47 #include <sys/strsun.h> 48 #include <sys/note.h> 49 #include <sys/crypto/common.h> 50 #include <sys/crypto/spi.h> 51 #include <sys/ddifm.h> 52 #include <sys/fm/protocol.h> 53 #include <sys/fm/util.h> 54 #include <sys/fm/io/ddi.h> 55 #include <sys/crypto/dca.h> 56 57 /* 58 * Core Deimos driver. 59 */ 60 61 static void dca_enlist2(dca_listnode_t *, dca_listnode_t *, 62 kmutex_t *); 63 static void dca_rmlist2(dca_listnode_t *node, kmutex_t *); 64 static dca_listnode_t *dca_delist2(dca_listnode_t *q, kmutex_t *); 65 static void dca_free_context_list(dca_t *dca); 66 static int dca_free_context_low(crypto_ctx_t *ctx); 67 static int dca_attach(dev_info_t *, ddi_attach_cmd_t); 68 static int dca_detach(dev_info_t *, ddi_detach_cmd_t); 69 static int dca_suspend(dca_t *); 70 static int dca_resume(dca_t *); 71 static int dca_init(dca_t *); 72 static int dca_reset(dca_t *, int); 73 static int dca_initworklist(dca_t *, dca_worklist_t *); 74 static void dca_uninit(dca_t *); 75 static void dca_initq(dca_listnode_t *); 76 static void dca_enqueue(dca_listnode_t *, dca_listnode_t *); 77 static dca_listnode_t *dca_dequeue(dca_listnode_t *); 78 static dca_listnode_t *dca_unqueue(dca_listnode_t *); 79 static dca_request_t *dca_newreq(dca_t *); 80 static dca_work_t *dca_getwork(dca_t *, int); 81 static void dca_freework(dca_work_t *); 82 static dca_work_t *dca_newwork(dca_t *); 83 static void dca_destroywork(dca_work_t *); 84 static void dca_schedule(dca_t *, int); 85 static void dca_reclaim(dca_t *, int); 86 static uint_t dca_intr(char *); 87 static void dca_failure(dca_t *, ddi_fault_location_t, 88 dca_fma_eclass_t index, uint64_t, int, char *, ...); 89 static void dca_jobtimeout(void *); 90 static int dca_drain(dca_t *); 91 static void dca_undrain(dca_t *); 92 static void dca_rejectjobs(dca_t *); 93 94 #ifdef SCHEDDELAY 95 static void dca_schedtimeout(void *); 96 #endif 97 98 /* 99 * We want these inlined for performance. 100 */ 101 #ifndef DEBUG 102 #pragma inline(dca_freereq, dca_getreq, dca_freework, dca_getwork) 103 #pragma inline(dca_enqueue, dca_dequeue, dca_rmqueue, dca_done) 104 #pragma inline(dca_reverse, dca_length) 105 #endif 106 107 /* 108 * Device operations. 109 */ 110 static struct dev_ops devops = { 111 DEVO_REV, /* devo_rev */ 112 0, /* devo_refcnt */ 113 nodev, /* devo_getinfo */ 114 nulldev, /* devo_identify */ 115 nulldev, /* devo_probe */ 116 dca_attach, /* devo_attach */ 117 dca_detach, /* devo_detach */ 118 nodev, /* devo_reset */ 119 NULL, /* devo_cb_ops */ 120 NULL, /* devo_bus_ops */ 121 ddi_power, /* devo_power */ 122 ddi_quiesce_not_supported, /* devo_quiesce */ 123 }; 124 125 #define IDENT "PCI Crypto Accelerator" 126 #define IDENT_SYM "Crypto Accel Sym 2.0" 127 #define IDENT_ASYM "Crypto Accel Asym 2.0" 128 129 /* Space-padded, will be filled in dynamically during registration */ 130 #define IDENT3 "PCI Crypto Accelerator Mod 2.0" 131 132 #define VENDOR "Sun Microsystems, Inc." 133 134 #define STALETIME (30 * SECOND) 135 136 #define crypto_prov_notify crypto_provider_notification 137 /* A 28 char function name doesn't leave much line space */ 138 139 /* 140 * Module linkage. 141 */ 142 static struct modldrv modldrv = { 143 &mod_driverops, /* drv_modops */ 144 IDENT, /* drv_linkinfo */ 145 &devops, /* drv_dev_ops */ 146 }; 147 148 extern struct mod_ops mod_cryptoops; 149 150 static struct modlcrypto modlcrypto = { 151 &mod_cryptoops, 152 IDENT3 153 }; 154 155 static struct modlinkage modlinkage = { 156 MODREV_1, /* ml_rev */ 157 &modldrv, /* ml_linkage */ 158 &modlcrypto, 159 NULL 160 }; 161 162 /* 163 * CSPI information (entry points, provider info, etc.) 164 */ 165 166 /* Mechanisms for the symmetric cipher provider */ 167 static crypto_mech_info_t dca_mech_info_tab1[] = { 168 /* DES-CBC */ 169 {SUN_CKM_DES_CBC, DES_CBC_MECH_INFO_TYPE, 170 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | 171 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC, 172 DES_KEY_LEN, DES_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES}, 173 /* 3DES-CBC */ 174 {SUN_CKM_DES3_CBC, DES3_CBC_MECH_INFO_TYPE, 175 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | 176 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC, 177 DES3_MIN_KEY_LEN, DES3_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES} 178 }; 179 180 /* Mechanisms for the asymmetric cipher provider */ 181 static crypto_mech_info_t dca_mech_info_tab2[] = { 182 /* DSA */ 183 {SUN_CKM_DSA, DSA_MECH_INFO_TYPE, 184 CRYPTO_FG_SIGN | CRYPTO_FG_VERIFY | 185 CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_VERIFY_ATOMIC, 186 CRYPTO_BYTES2BITS(DSA_MIN_KEY_LEN), 187 CRYPTO_BYTES2BITS(DSA_MAX_KEY_LEN), 188 CRYPTO_KEYSIZE_UNIT_IN_BITS}, 189 190 /* RSA */ 191 {SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE, 192 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN | 193 CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY | 194 CRYPTO_FG_VERIFY_RECOVER | 195 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC | 196 CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC | 197 CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC, 198 CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN), 199 CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN), 200 CRYPTO_KEYSIZE_UNIT_IN_BITS}, 201 {SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE, 202 CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN | 203 CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY | 204 CRYPTO_FG_VERIFY_RECOVER | 205 CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC | 206 CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC | 207 CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC, 208 CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN), 209 CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN), 210 CRYPTO_KEYSIZE_UNIT_IN_BITS} 211 }; 212 213 static void dca_provider_status(crypto_provider_handle_t, uint_t *); 214 215 static crypto_control_ops_t dca_control_ops = { 216 dca_provider_status 217 }; 218 219 static int dca_encrypt_init(crypto_ctx_t *, crypto_mechanism_t *, 220 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 221 static int dca_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, 222 crypto_req_handle_t); 223 static int dca_encrypt_update(crypto_ctx_t *, crypto_data_t *, 224 crypto_data_t *, crypto_req_handle_t); 225 static int dca_encrypt_final(crypto_ctx_t *, crypto_data_t *, 226 crypto_req_handle_t); 227 static int dca_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t, 228 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, 229 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 230 231 static int dca_decrypt_init(crypto_ctx_t *, crypto_mechanism_t *, 232 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 233 static int dca_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, 234 crypto_req_handle_t); 235 static int dca_decrypt_update(crypto_ctx_t *, crypto_data_t *, 236 crypto_data_t *, crypto_req_handle_t); 237 static int dca_decrypt_final(crypto_ctx_t *, crypto_data_t *, 238 crypto_req_handle_t); 239 static int dca_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t, 240 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, 241 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 242 243 static crypto_cipher_ops_t dca_cipher_ops = { 244 dca_encrypt_init, 245 dca_encrypt, 246 dca_encrypt_update, 247 dca_encrypt_final, 248 dca_encrypt_atomic, 249 dca_decrypt_init, 250 dca_decrypt, 251 dca_decrypt_update, 252 dca_decrypt_final, 253 dca_decrypt_atomic 254 }; 255 256 static int dca_sign_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, 257 crypto_spi_ctx_template_t, crypto_req_handle_t); 258 static int dca_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, 259 crypto_req_handle_t); 260 static int dca_sign_update(crypto_ctx_t *, crypto_data_t *, 261 crypto_req_handle_t); 262 static int dca_sign_final(crypto_ctx_t *, crypto_data_t *, 263 crypto_req_handle_t); 264 static int dca_sign_atomic(crypto_provider_handle_t, crypto_session_id_t, 265 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, 266 crypto_spi_ctx_template_t, crypto_req_handle_t); 267 static int dca_sign_recover_init(crypto_ctx_t *, crypto_mechanism_t *, 268 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 269 static int dca_sign_recover(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, 270 crypto_req_handle_t); 271 static int dca_sign_recover_atomic(crypto_provider_handle_t, 272 crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, 273 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 274 275 static crypto_sign_ops_t dca_sign_ops = { 276 dca_sign_init, 277 dca_sign, 278 dca_sign_update, 279 dca_sign_final, 280 dca_sign_atomic, 281 dca_sign_recover_init, 282 dca_sign_recover, 283 dca_sign_recover_atomic 284 }; 285 286 static int dca_verify_init(crypto_ctx_t *, crypto_mechanism_t *, 287 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 288 static int dca_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, 289 crypto_req_handle_t); 290 static int dca_verify_update(crypto_ctx_t *, crypto_data_t *, 291 crypto_req_handle_t); 292 static int dca_verify_final(crypto_ctx_t *, crypto_data_t *, 293 crypto_req_handle_t); 294 static int dca_verify_atomic(crypto_provider_handle_t, crypto_session_id_t, 295 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, 296 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 297 static int dca_verify_recover_init(crypto_ctx_t *, crypto_mechanism_t *, 298 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 299 static int dca_verify_recover(crypto_ctx_t *, crypto_data_t *, 300 crypto_data_t *, crypto_req_handle_t); 301 static int dca_verify_recover_atomic(crypto_provider_handle_t, 302 crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, 303 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); 304 305 static crypto_verify_ops_t dca_verify_ops = { 306 dca_verify_init, 307 dca_verify, 308 dca_verify_update, 309 dca_verify_final, 310 dca_verify_atomic, 311 dca_verify_recover_init, 312 dca_verify_recover, 313 dca_verify_recover_atomic 314 }; 315 316 static int dca_generate_random(crypto_provider_handle_t, crypto_session_id_t, 317 uchar_t *, size_t, crypto_req_handle_t); 318 319 static crypto_random_number_ops_t dca_random_number_ops = { 320 NULL, 321 dca_generate_random 322 }; 323 324 static int ext_info_sym(crypto_provider_handle_t prov, 325 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq); 326 static int ext_info_asym(crypto_provider_handle_t prov, 327 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq); 328 static int ext_info_base(crypto_provider_handle_t prov, 329 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id); 330 331 static crypto_provider_management_ops_t dca_provmanage_ops_1 = { 332 ext_info_sym, /* ext_info */ 333 NULL, /* init_token */ 334 NULL, /* init_pin */ 335 NULL /* set_pin */ 336 }; 337 338 static crypto_provider_management_ops_t dca_provmanage_ops_2 = { 339 ext_info_asym, /* ext_info */ 340 NULL, /* init_token */ 341 NULL, /* init_pin */ 342 NULL /* set_pin */ 343 }; 344 345 int dca_free_context(crypto_ctx_t *); 346 347 static crypto_ctx_ops_t dca_ctx_ops = { 348 NULL, 349 dca_free_context 350 }; 351 352 /* Operations for the symmetric cipher provider */ 353 static crypto_ops_t dca_crypto_ops1 = { 354 &dca_control_ops, 355 NULL, /* digest_ops */ 356 &dca_cipher_ops, 357 NULL, /* mac_ops */ 358 NULL, /* sign_ops */ 359 NULL, /* verify_ops */ 360 NULL, /* dual_ops */ 361 NULL, /* cipher_mac_ops */ 362 NULL, /* random_number_ops */ 363 NULL, /* session_ops */ 364 NULL, /* object_ops */ 365 NULL, /* key_ops */ 366 &dca_provmanage_ops_1, /* management_ops */ 367 &dca_ctx_ops 368 }; 369 370 /* Operations for the asymmetric cipher provider */ 371 static crypto_ops_t dca_crypto_ops2 = { 372 &dca_control_ops, 373 NULL, /* digest_ops */ 374 &dca_cipher_ops, 375 NULL, /* mac_ops */ 376 &dca_sign_ops, 377 &dca_verify_ops, 378 NULL, /* dual_ops */ 379 NULL, /* cipher_mac_ops */ 380 &dca_random_number_ops, 381 NULL, /* session_ops */ 382 NULL, /* object_ops */ 383 NULL, /* key_ops */ 384 &dca_provmanage_ops_2, /* management_ops */ 385 &dca_ctx_ops 386 }; 387 388 /* Provider information for the symmetric cipher provider */ 389 static crypto_provider_info_t dca_prov_info1 = { 390 CRYPTO_SPI_VERSION_1, 391 NULL, /* pi_provider_description */ 392 CRYPTO_HW_PROVIDER, 393 NULL, /* pi_provider_dev */ 394 NULL, /* pi_provider_handle */ 395 &dca_crypto_ops1, 396 sizeof (dca_mech_info_tab1)/sizeof (crypto_mech_info_t), 397 dca_mech_info_tab1, 398 0, /* pi_logical_provider_count */ 399 NULL /* pi_logical_providers */ 400 }; 401 402 /* Provider information for the asymmetric cipher provider */ 403 static crypto_provider_info_t dca_prov_info2 = { 404 CRYPTO_SPI_VERSION_1, 405 NULL, /* pi_provider_description */ 406 CRYPTO_HW_PROVIDER, 407 NULL, /* pi_provider_dev */ 408 NULL, /* pi_provider_handle */ 409 &dca_crypto_ops2, 410 sizeof (dca_mech_info_tab2)/sizeof (crypto_mech_info_t), 411 dca_mech_info_tab2, 412 0, /* pi_logical_provider_count */ 413 NULL /* pi_logical_providers */ 414 }; 415 416 /* Convenience macros */ 417 #define DCA_SOFTC_FROM_CTX(ctx) ((dca_t *)(ctx)->cc_provider) 418 #define DCA_MECH_FROM_CTX(ctx) \ 419 (((dca_request_t *)(ctx)->cc_provider_private)->dr_ctx.ctx_cm_type) 420 421 static int dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset, 422 caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags, 423 dca_chain_t *head, int *n_chain); 424 static uint64_t dca_ena(uint64_t ena); 425 static caddr_t dca_bufdaddr_out(crypto_data_t *data); 426 static char *dca_fma_eclass_string(char *model, dca_fma_eclass_t index); 427 static int dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle, 428 dca_fma_eclass_t eclass_index); 429 430 static void dca_fma_init(dca_t *dca); 431 static void dca_fma_fini(dca_t *dca); 432 static int dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, 433 const void *impl_data); 434 435 436 static dca_device_t dca_devices[] = { 437 /* Broadcom vanilla variants */ 438 { 0x14e4, 0x5820, "Broadcom 5820" }, 439 { 0x14e4, 0x5821, "Broadcom 5821" }, 440 { 0x14e4, 0x5822, "Broadcom 5822" }, 441 { 0x14e4, 0x5825, "Broadcom 5825" }, 442 /* Sun specific OEMd variants */ 443 { 0x108e, 0x5454, "SCA" }, 444 { 0x108e, 0x5455, "SCA 1000" }, 445 { 0x108e, 0x5457, "SCA 500" }, 446 /* subsysid should be 0x5457, but got 0x1 from HW. Assume both here. */ 447 { 0x108e, 0x1, "SCA 500" }, 448 }; 449 450 /* 451 * Device attributes. 452 */ 453 static struct ddi_device_acc_attr dca_regsattr = { 454 DDI_DEVICE_ATTR_V1, 455 DDI_STRUCTURE_LE_ACC, 456 DDI_STRICTORDER_ACC, 457 DDI_FLAGERR_ACC 458 }; 459 460 static struct ddi_device_acc_attr dca_devattr = { 461 DDI_DEVICE_ATTR_V0, 462 DDI_STRUCTURE_LE_ACC, 463 DDI_STRICTORDER_ACC 464 }; 465 466 static struct ddi_device_acc_attr dca_bufattr = { 467 DDI_DEVICE_ATTR_V0, 468 DDI_NEVERSWAP_ACC, 469 DDI_STRICTORDER_ACC 470 }; 471 472 static struct ddi_dma_attr dca_dmaattr = { 473 DMA_ATTR_V0, /* dma_attr_version */ 474 0x0, /* dma_attr_addr_lo */ 475 0xffffffffUL, /* dma_attr_addr_hi */ 476 0x00ffffffUL, /* dma_attr_count_max */ 477 0x40, /* dma_attr_align */ 478 0x40, /* dma_attr_burstsizes */ 479 0x1, /* dma_attr_minxfer */ 480 0x00ffffffUL, /* dma_attr_maxxfer */ 481 0xffffffffUL, /* dma_attr_seg */ 482 #if defined(__x86) 483 512, /* dma_attr_sgllen */ 484 #else 485 1, /* dma_attr_sgllen */ 486 #endif 487 1, /* dma_attr_granular */ 488 DDI_DMA_FLAGERR /* dma_attr_flags */ 489 }; 490 491 static void *dca_state = NULL; 492 int dca_mindma = 2500; 493 494 /* 495 * FMA eclass string definitions. Note that these string arrays must be 496 * consistent with the dca_fma_eclass_t enum. 497 */ 498 static char *dca_fma_eclass_sca1000[] = { 499 "sca1000.hw.device", 500 "sca1000.hw.timeout", 501 "sca1000.none" 502 }; 503 504 static char *dca_fma_eclass_sca500[] = { 505 "sca500.hw.device", 506 "sca500.hw.timeout", 507 "sca500.none" 508 }; 509 510 /* 511 * DDI entry points. 512 */ 513 int 514 _init(void) 515 { 516 int rv; 517 518 DBG(NULL, DMOD, "dca: in _init"); 519 520 if ((rv = ddi_soft_state_init(&dca_state, sizeof (dca_t), 1)) != 0) { 521 /* this should *never* happen! */ 522 return (rv); 523 } 524 525 if ((rv = mod_install(&modlinkage)) != 0) { 526 /* cleanup here */ 527 ddi_soft_state_fini(&dca_state); 528 return (rv); 529 } 530 531 return (0); 532 } 533 534 int 535 _fini(void) 536 { 537 int rv; 538 539 DBG(NULL, DMOD, "dca: in _fini"); 540 541 if ((rv = mod_remove(&modlinkage)) == 0) { 542 /* cleanup here */ 543 ddi_soft_state_fini(&dca_state); 544 } 545 return (rv); 546 } 547 548 int 549 _info(struct modinfo *modinfop) 550 { 551 DBG(NULL, DMOD, "dca: in _info"); 552 553 return (mod_info(&modlinkage, modinfop)); 554 } 555 556 int 557 dca_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 558 { 559 ddi_acc_handle_t pci; 560 int instance; 561 ddi_iblock_cookie_t ibc; 562 int intr_added = 0; 563 dca_t *dca; 564 ushort_t venid; 565 ushort_t devid; 566 ushort_t revid; 567 ushort_t subsysid; 568 ushort_t subvenid; 569 int i; 570 int ret; 571 char ID[64]; 572 static char *unknowndev = "Unknown device"; 573 574 #if DEBUG 575 /* these are only used for debugging */ 576 ushort_t pcicomm; 577 ushort_t pcistat; 578 uchar_t cachelinesz; 579 uchar_t mingnt; 580 uchar_t maxlat; 581 uchar_t lattmr; 582 #endif 583 584 instance = ddi_get_instance(dip); 585 586 DBG(NULL, DMOD, "dca: in dca_attach() for %d", instance); 587 588 switch (cmd) { 589 case DDI_RESUME: 590 if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) { 591 dca_diperror(dip, "no soft state in detach"); 592 return (DDI_FAILURE); 593 } 594 /* assumption: we won't be DDI_DETACHed until we return */ 595 return (dca_resume(dca)); 596 case DDI_ATTACH: 597 break; 598 default: 599 return (DDI_FAILURE); 600 } 601 602 if (ddi_slaveonly(dip) == DDI_SUCCESS) { 603 dca_diperror(dip, "slot does not support PCI bus-master"); 604 return (DDI_FAILURE); 605 } 606 607 if (ddi_intr_hilevel(dip, 0) != 0) { 608 dca_diperror(dip, "hilevel interrupts not supported"); 609 return (DDI_FAILURE); 610 } 611 612 if (pci_config_setup(dip, &pci) != DDI_SUCCESS) { 613 dca_diperror(dip, "unable to setup PCI config handle"); 614 return (DDI_FAILURE); 615 } 616 617 /* common PCI attributes */ 618 venid = pci_config_get16(pci, PCI_VENID); 619 devid = pci_config_get16(pci, PCI_DEVID); 620 revid = pci_config_get8(pci, PCI_REVID); 621 subvenid = pci_config_get16(pci, PCI_SUBVENID); 622 subsysid = pci_config_get16(pci, PCI_SUBSYSID); 623 624 /* 625 * Broadcom-specific timings. 626 * We disable these timers/counters since they can cause 627 * incorrect false failures when the bus is just a little 628 * bit slow, or busy. 629 */ 630 pci_config_put8(pci, PCI_TRDYTO, 0); 631 pci_config_put8(pci, PCI_RETRIES, 0); 632 633 /* initialize PCI access settings */ 634 pci_config_put16(pci, PCI_COMM, PCICOMM_SEE | 635 PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE); 636 637 /* set up our PCI latency timer */ 638 pci_config_put8(pci, PCI_LATTMR, 0x40); 639 640 #if DEBUG 641 /* read registers (for debugging) */ 642 pcicomm = pci_config_get16(pci, PCI_COMM); 643 pcistat = pci_config_get16(pci, PCI_STATUS); 644 cachelinesz = pci_config_get8(pci, PCI_CACHELINESZ); 645 mingnt = pci_config_get8(pci, PCI_MINGNT); 646 maxlat = pci_config_get8(pci, PCI_MAXLAT); 647 lattmr = pci_config_get8(pci, PCI_LATTMR); 648 #endif 649 650 pci_config_teardown(&pci); 651 652 if (ddi_get_iblock_cookie(dip, 0, &ibc) != DDI_SUCCESS) { 653 dca_diperror(dip, "unable to get iblock cookie"); 654 return (DDI_FAILURE); 655 } 656 657 if (ddi_soft_state_zalloc(dca_state, instance) != DDI_SUCCESS) { 658 dca_diperror(dip, "unable to allocate soft state"); 659 return (DDI_FAILURE); 660 } 661 662 dca = ddi_get_soft_state(dca_state, instance); 663 ASSERT(dca != NULL); 664 dca->dca_dip = dip; 665 WORKLIST(dca, MCR1)->dwl_prov = 0; 666 WORKLIST(dca, MCR2)->dwl_prov = 0; 667 /* figure pagesize */ 668 dca->dca_pagesize = ddi_ptob(dip, 1); 669 670 /* 671 * Search for the device in our supported devices table. This 672 * is here for two reasons. First, we want to ensure that 673 * only Sun-qualified (and presumably Sun-labeled) devices can 674 * be used with this driver. Second, some devices have 675 * specific differences. E.g. the 5821 has support for a 676 * special mode of RC4, deeper queues, power management, and 677 * other changes. Also, the export versions of some of these 678 * chips don't support RC4 or 3DES, so we catch that here. 679 * 680 * Note that we only look at the upper nibble of the device 681 * id, which is used to distinguish export vs. domestic 682 * versions of the chip. (The lower nibble is used for 683 * stepping information.) 684 */ 685 for (i = 0; i < (sizeof (dca_devices) / sizeof (dca_device_t)); i++) { 686 /* 687 * Try to match the subsystem information first. 688 */ 689 if (subvenid && (subvenid == dca_devices[i].dd_vendor_id) && 690 subsysid && (subsysid == dca_devices[i].dd_device_id)) { 691 dca->dca_model = dca_devices[i].dd_model; 692 dca->dca_devid = dca_devices[i].dd_device_id; 693 break; 694 } 695 /* 696 * Failing that, try the generic vendor and device id. 697 * Even if we find a match, we keep searching anyway, 698 * since we would prefer to find a match based on the 699 * subsystem ids. 700 */ 701 if ((venid == dca_devices[i].dd_vendor_id) && 702 (devid == dca_devices[i].dd_device_id)) { 703 dca->dca_model = dca_devices[i].dd_model; 704 dca->dca_devid = dca_devices[i].dd_device_id; 705 } 706 } 707 /* try and handle an unrecognized device */ 708 if (dca->dca_model == NULL) { 709 dca->dca_model = unknowndev; 710 dca_error(dca, "device not recognized, not supported"); 711 DBG(dca, DPCI, "i=%d venid=%x devid=%x rev=%d", 712 i, venid, devid, revid); 713 } 714 715 if (ddi_prop_update_string(DDI_DEV_T_NONE, dip, "description", 716 dca->dca_model) != DDI_SUCCESS) { 717 dca_error(dca, "unable to create description property"); 718 return (DDI_FAILURE); 719 } 720 721 DBG(dca, DPCI, "PCI command=0x%x status=%x cachelinesz=%x", 722 pcicomm, pcistat, cachelinesz); 723 DBG(dca, DPCI, "mingnt=0x%x maxlat=0x%x lattmr=0x%x", 724 mingnt, maxlat, lattmr); 725 726 /* 727 * initialize locks, etc. 728 */ 729 (void) mutex_init(&dca->dca_intrlock, NULL, MUTEX_DRIVER, ibc); 730 731 /* use RNGSHA1 by default */ 732 if (ddi_getprop(DDI_DEV_T_ANY, dip, 733 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "rngdirect", 0) == 0) { 734 dca->dca_flags |= DCA_RNGSHA1; 735 } 736 737 /* initialize FMA */ 738 dca_fma_init(dca); 739 740 /* initialize some key data structures */ 741 if (dca_init(dca) != DDI_SUCCESS) { 742 goto failed; 743 } 744 745 /* initialize kstats */ 746 dca_ksinit(dca); 747 748 /* setup access to registers */ 749 if (ddi_regs_map_setup(dip, 1, (caddr_t *)&dca->dca_regs, 750 0, 0, &dca_regsattr, &dca->dca_regs_handle) != DDI_SUCCESS) { 751 dca_error(dca, "unable to map registers"); 752 goto failed; 753 } 754 755 DBG(dca, DCHATTY, "MCR1 = %x", GETCSR(dca, CSR_MCR1)); 756 DBG(dca, DCHATTY, "CONTROL = %x", GETCSR(dca, CSR_DMACTL)); 757 DBG(dca, DCHATTY, "STATUS = %x", GETCSR(dca, CSR_DMASTAT)); 758 DBG(dca, DCHATTY, "DMAEA = %x", GETCSR(dca, CSR_DMAEA)); 759 DBG(dca, DCHATTY, "MCR2 = %x", GETCSR(dca, CSR_MCR2)); 760 761 /* reset the chip */ 762 if (dca_reset(dca, 0) < 0) { 763 goto failed; 764 } 765 766 /* initialize the chip */ 767 PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64); 768 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 769 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 770 goto failed; 771 } 772 773 /* add the interrupt */ 774 if (ddi_add_intr(dip, 0, &dca->dca_icookie, NULL, dca_intr, 775 (void *)dca) != DDI_SUCCESS) { 776 DBG(dca, DWARN, "ddi_add_intr failed"); 777 goto failed; 778 } else { 779 intr_added = 1; 780 } 781 782 /* enable interrupts on the device */ 783 /* 784 * XXX: Note, 5820A1 errata indicates that this may clobber 785 * bits 24 and 23, which affect the speed of the RNG. Since 786 * we always want to run in full-speed mode, this should be 787 * harmless. 788 */ 789 if (dca->dca_devid == 0x5825) { 790 /* for 5825 - increase the DMA read size */ 791 SETBIT(dca, CSR_DMACTL, 792 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256); 793 } else { 794 SETBIT(dca, CSR_DMACTL, 795 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE); 796 } 797 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 798 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 799 goto failed; 800 } 801 802 /* register MCR1 with the crypto framework */ 803 /* Be careful not to exceed 32 chars */ 804 (void) sprintf(ID, "%s/%d %s", 805 ddi_driver_name(dip), ddi_get_instance(dip), IDENT_SYM); 806 dca_prov_info1.pi_provider_description = ID; 807 dca_prov_info1.pi_provider_dev.pd_hw = dip; 808 dca_prov_info1.pi_provider_handle = dca; 809 if ((ret = crypto_register_provider(&dca_prov_info1, 810 &WORKLIST(dca, MCR1)->dwl_prov)) != CRYPTO_SUCCESS) { 811 cmn_err(CE_WARN, 812 "crypto_register_provider() failed (%d) for MCR1", ret); 813 goto failed; 814 } 815 816 /* register MCR2 with the crypto framework */ 817 /* Be careful not to exceed 32 chars */ 818 (void) sprintf(ID, "%s/%d %s", 819 ddi_driver_name(dip), ddi_get_instance(dip), IDENT_ASYM); 820 dca_prov_info2.pi_provider_description = ID; 821 dca_prov_info2.pi_provider_dev.pd_hw = dip; 822 dca_prov_info2.pi_provider_handle = dca; 823 if ((ret = crypto_register_provider(&dca_prov_info2, 824 &WORKLIST(dca, MCR2)->dwl_prov)) != CRYPTO_SUCCESS) { 825 cmn_err(CE_WARN, 826 "crypto_register_provider() failed (%d) for MCR2", ret); 827 goto failed; 828 } 829 830 crypto_prov_notify(WORKLIST(dca, MCR1)->dwl_prov, 831 CRYPTO_PROVIDER_READY); 832 crypto_prov_notify(WORKLIST(dca, MCR2)->dwl_prov, 833 CRYPTO_PROVIDER_READY); 834 835 /* Initialize the local random number pool for this instance */ 836 if ((ret = dca_random_init(dca)) != CRYPTO_SUCCESS) { 837 goto failed; 838 } 839 840 mutex_enter(&dca->dca_intrlock); 841 dca->dca_jobtid = timeout(dca_jobtimeout, (void *)dca, 842 drv_usectohz(SECOND)); 843 mutex_exit(&dca->dca_intrlock); 844 845 ddi_set_driver_private(dip, (caddr_t)dca); 846 847 ddi_report_dev(dip); 848 849 if (ddi_get_devstate(dca->dca_dip) != DDI_DEVSTATE_UP) { 850 ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_RESTORED); 851 } 852 853 return (DDI_SUCCESS); 854 855 failed: 856 /* unregister from the crypto framework */ 857 if (WORKLIST(dca, MCR1)->dwl_prov != 0) { 858 (void) crypto_unregister_provider( 859 WORKLIST(dca, MCR1)->dwl_prov); 860 } 861 if (WORKLIST(dca, MCR2)->dwl_prov != 0) { 862 (void) crypto_unregister_provider( 863 WORKLIST(dca, MCR2)->dwl_prov); 864 } 865 if (intr_added) { 866 CLRBIT(dca, CSR_DMACTL, 867 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE); 868 /* unregister intr handler */ 869 ddi_remove_intr(dip, 0, dca->dca_icookie); 870 } 871 if (dca->dca_regs_handle) { 872 ddi_regs_map_free(&dca->dca_regs_handle); 873 } 874 if (dca->dca_intrstats) { 875 kstat_delete(dca->dca_intrstats); 876 } 877 if (dca->dca_ksp) { 878 kstat_delete(dca->dca_ksp); 879 } 880 dca_uninit(dca); 881 882 /* finalize FMA */ 883 dca_fma_fini(dca); 884 885 mutex_destroy(&dca->dca_intrlock); 886 ddi_soft_state_free(dca_state, instance); 887 return (DDI_FAILURE); 888 889 } 890 891 int 892 dca_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 893 { 894 int instance; 895 dca_t *dca; 896 timeout_id_t tid; 897 898 instance = ddi_get_instance(dip); 899 900 DBG(NULL, DMOD, "dca: in dca_detach() for %d", instance); 901 902 switch (cmd) { 903 case DDI_SUSPEND: 904 if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) { 905 dca_diperror(dip, "no soft state in detach"); 906 return (DDI_FAILURE); 907 } 908 /* assumption: we won't be DDI_DETACHed until we return */ 909 return (dca_suspend(dca)); 910 911 case DDI_DETACH: 912 break; 913 default: 914 return (DDI_FAILURE); 915 } 916 917 if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) { 918 dca_diperror(dip, "no soft state in detach"); 919 return (DDI_FAILURE); 920 } 921 922 /* 923 * Unregister from kCF. 924 * This needs to be done at the beginning of detach. 925 */ 926 if (WORKLIST(dca, MCR1)->dwl_prov != 0) { 927 if (crypto_unregister_provider( 928 WORKLIST(dca, MCR1)->dwl_prov) != CRYPTO_SUCCESS) { 929 dca_error(dca, "unable to unregister MCR1 from kcf"); 930 return (DDI_FAILURE); 931 } 932 } 933 934 if (WORKLIST(dca, MCR2)->dwl_prov != 0) { 935 if (crypto_unregister_provider( 936 WORKLIST(dca, MCR2)->dwl_prov) != CRYPTO_SUCCESS) { 937 dca_error(dca, "unable to unregister MCR2 from kcf"); 938 return (DDI_FAILURE); 939 } 940 } 941 942 /* 943 * Cleanup the private context list. Once the 944 * crypto_unregister_provider returns, it is safe to do so. 945 */ 946 dca_free_context_list(dca); 947 948 /* Cleanup the local random number pool */ 949 dca_random_fini(dca); 950 951 /* send any jobs in the waitq back to kCF */ 952 dca_rejectjobs(dca); 953 954 /* untimeout the timeouts */ 955 mutex_enter(&dca->dca_intrlock); 956 tid = dca->dca_jobtid; 957 dca->dca_jobtid = 0; 958 mutex_exit(&dca->dca_intrlock); 959 if (tid) { 960 (void) untimeout(tid); 961 } 962 963 /* disable device interrupts */ 964 CLRBIT(dca, CSR_DMACTL, DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE); 965 966 /* unregister interrupt handlers */ 967 ddi_remove_intr(dip, 0, dca->dca_icookie); 968 969 /* release our regs handle */ 970 ddi_regs_map_free(&dca->dca_regs_handle); 971 972 /* toss out kstats */ 973 if (dca->dca_intrstats) { 974 kstat_delete(dca->dca_intrstats); 975 } 976 if (dca->dca_ksp) { 977 kstat_delete(dca->dca_ksp); 978 } 979 980 mutex_destroy(&dca->dca_intrlock); 981 dca_uninit(dca); 982 983 /* finalize FMA */ 984 dca_fma_fini(dca); 985 986 ddi_soft_state_free(dca_state, instance); 987 988 return (DDI_SUCCESS); 989 } 990 991 int 992 dca_resume(dca_t *dca) 993 { 994 ddi_acc_handle_t pci; 995 996 if (pci_config_setup(dca->dca_dip, &pci) != DDI_SUCCESS) { 997 dca_error(dca, "unable to setup PCI config handle"); 998 return (DDI_FAILURE); 999 } 1000 1001 /* 1002 * Reprogram registers in PCI configuration space. 1003 */ 1004 1005 /* Broadcom-specific timers -- we disable them. */ 1006 pci_config_put8(pci, PCI_TRDYTO, 0); 1007 pci_config_put8(pci, PCI_RETRIES, 0); 1008 1009 /* initialize PCI access settings */ 1010 pci_config_put16(pci, PCI_COMM, PCICOMM_SEE | 1011 PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE); 1012 1013 /* set up our PCI latency timer */ 1014 pci_config_put8(pci, PCI_LATTMR, 0x40); 1015 1016 pci_config_teardown(&pci); 1017 1018 if (dca_reset(dca, 0) < 0) { 1019 dca_error(dca, "unable to reset device during resume"); 1020 return (DDI_FAILURE); 1021 } 1022 1023 /* 1024 * Now restore the card-specific CSRs. 1025 */ 1026 1027 /* restore endianness settings */ 1028 PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64); 1029 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 1030 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) 1031 return (DDI_FAILURE); 1032 1033 /* restore interrupt enables */ 1034 if (dca->dca_devid == 0x5825) { 1035 /* for 5825 set 256 byte read size to improve performance */ 1036 SETBIT(dca, CSR_DMACTL, 1037 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256); 1038 } else { 1039 SETBIT(dca, CSR_DMACTL, 1040 DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE); 1041 } 1042 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 1043 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) 1044 return (DDI_FAILURE); 1045 1046 /* resume scheduling jobs on the device */ 1047 dca_undrain(dca); 1048 1049 return (DDI_SUCCESS); 1050 } 1051 1052 int 1053 dca_suspend(dca_t *dca) 1054 { 1055 if ((dca_drain(dca)) != 0) { 1056 return (DDI_FAILURE); 1057 } 1058 if (dca_reset(dca, 0) < 0) { 1059 dca_error(dca, "unable to reset device during suspend"); 1060 return (DDI_FAILURE); 1061 } 1062 return (DDI_SUCCESS); 1063 } 1064 1065 /* 1066 * Hardware access stuff. 1067 */ 1068 int 1069 dca_reset(dca_t *dca, int failreset) 1070 { 1071 int i; 1072 1073 if (dca->dca_regs_handle == NULL) { 1074 return (-1); 1075 } 1076 1077 PUTCSR(dca, CSR_DMACTL, DMACTL_RESET); 1078 if (!failreset) { 1079 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 1080 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) 1081 return (-1); 1082 } 1083 1084 /* now wait for a reset */ 1085 for (i = 1; i < 100; i++) { 1086 uint32_t dmactl; 1087 drv_usecwait(100); 1088 dmactl = GETCSR(dca, CSR_DMACTL); 1089 if (!failreset) { 1090 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 1091 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) 1092 return (-1); 1093 } 1094 if ((dmactl & DMACTL_RESET) == 0) { 1095 DBG(dca, DCHATTY, "reset in %d usec", i * 100); 1096 return (0); 1097 } 1098 } 1099 if (!failreset) { 1100 dca_failure(dca, DDI_DEVICE_FAULT, 1101 DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR, 1102 "timeout waiting for reset after %d usec", i * 100); 1103 } 1104 return (-1); 1105 } 1106 1107 int 1108 dca_initworklist(dca_t *dca, dca_worklist_t *wlp) 1109 { 1110 int i; 1111 int reqprealloc = wlp->dwl_hiwater + (MAXWORK * MAXREQSPERMCR); 1112 1113 /* 1114 * Set up work queue. 1115 */ 1116 mutex_init(&wlp->dwl_lock, NULL, MUTEX_DRIVER, dca->dca_icookie); 1117 mutex_init(&wlp->dwl_freereqslock, NULL, MUTEX_DRIVER, 1118 dca->dca_icookie); 1119 mutex_init(&wlp->dwl_freelock, NULL, MUTEX_DRIVER, dca->dca_icookie); 1120 cv_init(&wlp->dwl_cv, NULL, CV_DRIVER, NULL); 1121 1122 mutex_enter(&wlp->dwl_lock); 1123 1124 dca_initq(&wlp->dwl_freereqs); 1125 dca_initq(&wlp->dwl_waitq); 1126 dca_initq(&wlp->dwl_freework); 1127 dca_initq(&wlp->dwl_runq); 1128 1129 for (i = 0; i < MAXWORK; i++) { 1130 dca_work_t *workp; 1131 1132 if ((workp = dca_newwork(dca)) == NULL) { 1133 dca_error(dca, "unable to allocate work"); 1134 mutex_exit(&wlp->dwl_lock); 1135 return (DDI_FAILURE); 1136 } 1137 workp->dw_wlp = wlp; 1138 dca_freework(workp); 1139 } 1140 mutex_exit(&wlp->dwl_lock); 1141 1142 for (i = 0; i < reqprealloc; i++) { 1143 dca_request_t *reqp; 1144 1145 if ((reqp = dca_newreq(dca)) == NULL) { 1146 dca_error(dca, "unable to allocate request"); 1147 return (DDI_FAILURE); 1148 } 1149 reqp->dr_dca = dca; 1150 reqp->dr_wlp = wlp; 1151 dca_freereq(reqp); 1152 } 1153 return (DDI_SUCCESS); 1154 } 1155 1156 int 1157 dca_init(dca_t *dca) 1158 { 1159 dca_worklist_t *wlp; 1160 1161 /* Initialize the private context list and the corresponding lock. */ 1162 mutex_init(&dca->dca_ctx_list_lock, NULL, MUTEX_DRIVER, NULL); 1163 dca_initq(&dca->dca_ctx_list); 1164 1165 /* 1166 * MCR1 algorithms. 1167 */ 1168 wlp = WORKLIST(dca, MCR1); 1169 (void) sprintf(wlp->dwl_name, "dca%d:mcr1", 1170 ddi_get_instance(dca->dca_dip)); 1171 wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY, 1172 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, 1173 "mcr1_lowater", MCR1LOWATER); 1174 wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY, 1175 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, 1176 "mcr1_hiwater", MCR1HIWATER); 1177 wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY, 1178 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, 1179 "mcr1_maxreqs", MCR1MAXREQS), MAXREQSPERMCR); 1180 wlp->dwl_dca = dca; 1181 wlp->dwl_mcr = MCR1; 1182 if (dca_initworklist(dca, wlp) != DDI_SUCCESS) { 1183 return (DDI_FAILURE); 1184 } 1185 1186 /* 1187 * MCR2 algorithms. 1188 */ 1189 wlp = WORKLIST(dca, MCR2); 1190 (void) sprintf(wlp->dwl_name, "dca%d:mcr2", 1191 ddi_get_instance(dca->dca_dip)); 1192 wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY, 1193 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, 1194 "mcr2_lowater", MCR2LOWATER); 1195 wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY, 1196 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, 1197 "mcr2_hiwater", MCR2HIWATER); 1198 wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY, 1199 dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, 1200 "mcr2_maxreqs", MCR2MAXREQS), MAXREQSPERMCR); 1201 wlp->dwl_dca = dca; 1202 wlp->dwl_mcr = MCR2; 1203 if (dca_initworklist(dca, wlp) != DDI_SUCCESS) { 1204 return (DDI_FAILURE); 1205 } 1206 return (DDI_SUCCESS); 1207 } 1208 1209 /* 1210 * Uninitialize worklists. This routine should only be called when no 1211 * active jobs (hence DMA mappings) exist. One way to ensure this is 1212 * to unregister from kCF before calling this routine. (This is done 1213 * e.g. in detach(9e).) 1214 */ 1215 void 1216 dca_uninit(dca_t *dca) 1217 { 1218 int mcr; 1219 1220 mutex_destroy(&dca->dca_ctx_list_lock); 1221 1222 for (mcr = MCR1; mcr <= MCR2; mcr++) { 1223 dca_worklist_t *wlp = WORKLIST(dca, mcr); 1224 dca_work_t *workp; 1225 dca_request_t *reqp; 1226 1227 if (dca->dca_regs_handle == NULL) { 1228 continue; 1229 } 1230 1231 mutex_enter(&wlp->dwl_lock); 1232 while ((workp = dca_getwork(dca, mcr)) != NULL) { 1233 dca_destroywork(workp); 1234 } 1235 mutex_exit(&wlp->dwl_lock); 1236 while ((reqp = dca_getreq(dca, mcr, 0)) != NULL) { 1237 dca_destroyreq(reqp); 1238 } 1239 1240 mutex_destroy(&wlp->dwl_lock); 1241 mutex_destroy(&wlp->dwl_freereqslock); 1242 mutex_destroy(&wlp->dwl_freelock); 1243 cv_destroy(&wlp->dwl_cv); 1244 wlp->dwl_prov = 0; 1245 } 1246 } 1247 1248 static void 1249 dca_enlist2(dca_listnode_t *q, dca_listnode_t *node, kmutex_t *lock) 1250 { 1251 if (!q || !node) 1252 return; 1253 1254 mutex_enter(lock); 1255 node->dl_next2 = q; 1256 node->dl_prev2 = q->dl_prev2; 1257 node->dl_next2->dl_prev2 = node; 1258 node->dl_prev2->dl_next2 = node; 1259 mutex_exit(lock); 1260 } 1261 1262 static void 1263 dca_rmlist2(dca_listnode_t *node, kmutex_t *lock) 1264 { 1265 if (!node) 1266 return; 1267 1268 mutex_enter(lock); 1269 node->dl_next2->dl_prev2 = node->dl_prev2; 1270 node->dl_prev2->dl_next2 = node->dl_next2; 1271 node->dl_next2 = NULL; 1272 node->dl_prev2 = NULL; 1273 mutex_exit(lock); 1274 } 1275 1276 static dca_listnode_t * 1277 dca_delist2(dca_listnode_t *q, kmutex_t *lock) 1278 { 1279 dca_listnode_t *node; 1280 1281 mutex_enter(lock); 1282 if ((node = q->dl_next2) == q) { 1283 mutex_exit(lock); 1284 return (NULL); 1285 } 1286 1287 node->dl_next2->dl_prev2 = node->dl_prev2; 1288 node->dl_prev2->dl_next2 = node->dl_next2; 1289 node->dl_next2 = NULL; 1290 node->dl_prev2 = NULL; 1291 mutex_exit(lock); 1292 1293 return (node); 1294 } 1295 1296 void 1297 dca_initq(dca_listnode_t *q) 1298 { 1299 q->dl_next = q; 1300 q->dl_prev = q; 1301 q->dl_next2 = q; 1302 q->dl_prev2 = q; 1303 } 1304 1305 void 1306 dca_enqueue(dca_listnode_t *q, dca_listnode_t *node) 1307 { 1308 /* 1309 * Enqueue submits at the "tail" of the list, i.e. just 1310 * behind the sentinel. 1311 */ 1312 node->dl_next = q; 1313 node->dl_prev = q->dl_prev; 1314 node->dl_next->dl_prev = node; 1315 node->dl_prev->dl_next = node; 1316 } 1317 1318 void 1319 dca_rmqueue(dca_listnode_t *node) 1320 { 1321 node->dl_next->dl_prev = node->dl_prev; 1322 node->dl_prev->dl_next = node->dl_next; 1323 node->dl_next = NULL; 1324 node->dl_prev = NULL; 1325 } 1326 1327 dca_listnode_t * 1328 dca_dequeue(dca_listnode_t *q) 1329 { 1330 dca_listnode_t *node; 1331 /* 1332 * Dequeue takes from the "head" of the list, i.e. just after 1333 * the sentinel. 1334 */ 1335 if ((node = q->dl_next) == q) { 1336 /* queue is empty */ 1337 return (NULL); 1338 } 1339 dca_rmqueue(node); 1340 return (node); 1341 } 1342 1343 /* this is the opposite of dequeue, it takes things off in LIFO order */ 1344 dca_listnode_t * 1345 dca_unqueue(dca_listnode_t *q) 1346 { 1347 dca_listnode_t *node; 1348 /* 1349 * unqueue takes from the "tail" of the list, i.e. just before 1350 * the sentinel. 1351 */ 1352 if ((node = q->dl_prev) == q) { 1353 /* queue is empty */ 1354 return (NULL); 1355 } 1356 dca_rmqueue(node); 1357 return (node); 1358 } 1359 1360 dca_listnode_t * 1361 dca_peekqueue(dca_listnode_t *q) 1362 { 1363 dca_listnode_t *node; 1364 1365 if ((node = q->dl_next) == q) { 1366 return (NULL); 1367 } else { 1368 return (node); 1369 } 1370 } 1371 1372 /* 1373 * Interrupt service routine. 1374 */ 1375 uint_t 1376 dca_intr(char *arg) 1377 { 1378 dca_t *dca = (dca_t *)arg; 1379 uint32_t status; 1380 1381 mutex_enter(&dca->dca_intrlock); 1382 status = GETCSR(dca, CSR_DMASTAT); 1383 PUTCSR(dca, CSR_DMASTAT, status & DMASTAT_INTERRUPTS); 1384 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 1385 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 1386 mutex_exit(&dca->dca_intrlock); 1387 return ((uint_t)DDI_FAILURE); 1388 } 1389 1390 DBG(dca, DINTR, "interrupted, status = 0x%x!", status); 1391 1392 if ((status & DMASTAT_INTERRUPTS) == 0) { 1393 /* increment spurious interrupt kstat */ 1394 if (dca->dca_intrstats) { 1395 KIOIP(dca)->intrs[KSTAT_INTR_SPURIOUS]++; 1396 } 1397 mutex_exit(&dca->dca_intrlock); 1398 return (DDI_INTR_UNCLAIMED); 1399 } 1400 1401 if (dca->dca_intrstats) { 1402 KIOIP(dca)->intrs[KSTAT_INTR_HARD]++; 1403 } 1404 if (status & DMASTAT_MCR1INT) { 1405 DBG(dca, DINTR, "MCR1 interrupted"); 1406 mutex_enter(&(WORKLIST(dca, MCR1)->dwl_lock)); 1407 dca_schedule(dca, MCR1); 1408 dca_reclaim(dca, MCR1); 1409 mutex_exit(&(WORKLIST(dca, MCR1)->dwl_lock)); 1410 } 1411 1412 if (status & DMASTAT_MCR2INT) { 1413 DBG(dca, DINTR, "MCR2 interrupted"); 1414 mutex_enter(&(WORKLIST(dca, MCR2)->dwl_lock)); 1415 dca_schedule(dca, MCR2); 1416 dca_reclaim(dca, MCR2); 1417 mutex_exit(&(WORKLIST(dca, MCR2)->dwl_lock)); 1418 } 1419 1420 if (status & DMASTAT_ERRINT) { 1421 uint32_t erraddr; 1422 erraddr = GETCSR(dca, CSR_DMAEA); 1423 mutex_exit(&dca->dca_intrlock); 1424 1425 /* 1426 * bit 1 of the error address indicates failure during 1427 * read if set, during write otherwise. 1428 */ 1429 dca_failure(dca, DDI_DEVICE_FAULT, 1430 DCA_FM_ECLASS_HW_DEVICE, dca_ena(0), CRYPTO_DEVICE_ERROR, 1431 "DMA master access error %s address 0x%x", 1432 erraddr & 0x1 ? "reading" : "writing", erraddr & ~1); 1433 return (DDI_INTR_CLAIMED); 1434 } 1435 1436 mutex_exit(&dca->dca_intrlock); 1437 1438 return (DDI_INTR_CLAIMED); 1439 } 1440 1441 /* 1442 * Reverse a string of bytes from s1 into s2. The reversal happens 1443 * from the tail of s1. If len1 < len2, then null bytes will be 1444 * padded to the end of s2. If len2 < len1, then (presumably null) 1445 * bytes will be dropped from the start of s1. 1446 * 1447 * The rationale here is that when s1 (source) is shorter, then we 1448 * are reversing from big-endian ordering, into device ordering, and 1449 * want to add some extra nulls to the tail (MSB) side of the device. 1450 * 1451 * Similarly, when s2 (dest) is shorter, then we are truncating what 1452 * are presumably null MSB bits from the device. 1453 * 1454 * There is an expectation when reversing from the device back into 1455 * big-endian, that the number of bytes to reverse and the target size 1456 * will match, and no truncation or padding occurs. 1457 */ 1458 void 1459 dca_reverse(void *s1, void *s2, int len1, int len2) 1460 { 1461 caddr_t src, dst; 1462 1463 if (len1 == 0) { 1464 if (len2) { 1465 bzero(s2, len2); 1466 } 1467 return; 1468 } 1469 src = (caddr_t)s1 + len1 - 1; 1470 dst = s2; 1471 while ((src >= (caddr_t)s1) && (len2)) { 1472 *dst++ = *src--; 1473 len2--; 1474 } 1475 while (len2 > 0) { 1476 *dst++ = 0; 1477 len2--; 1478 } 1479 } 1480 1481 uint16_t 1482 dca_padfull(int num) 1483 { 1484 if (num <= 512) { 1485 return (BITS2BYTES(512)); 1486 } 1487 if (num <= 768) { 1488 return (BITS2BYTES(768)); 1489 } 1490 if (num <= 1024) { 1491 return (BITS2BYTES(1024)); 1492 } 1493 if (num <= 1536) { 1494 return (BITS2BYTES(1536)); 1495 } 1496 if (num <= 2048) { 1497 return (BITS2BYTES(2048)); 1498 } 1499 return (0); 1500 } 1501 1502 uint16_t 1503 dca_padhalf(int num) 1504 { 1505 if (num <= 256) { 1506 return (BITS2BYTES(256)); 1507 } 1508 if (num <= 384) { 1509 return (BITS2BYTES(384)); 1510 } 1511 if (num <= 512) { 1512 return (BITS2BYTES(512)); 1513 } 1514 if (num <= 768) { 1515 return (BITS2BYTES(768)); 1516 } 1517 if (num <= 1024) { 1518 return (BITS2BYTES(1024)); 1519 } 1520 return (0); 1521 } 1522 1523 dca_work_t * 1524 dca_newwork(dca_t *dca) 1525 { 1526 dca_work_t *workp; 1527 size_t size; 1528 ddi_dma_cookie_t c; 1529 unsigned nc; 1530 int rv; 1531 1532 workp = kmem_zalloc(sizeof (dca_work_t), KM_SLEEP); 1533 1534 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, 1535 DDI_DMA_SLEEP, NULL, &workp->dw_mcr_dmah); 1536 if (rv != 0) { 1537 dca_error(dca, "unable to alloc MCR DMA handle"); 1538 dca_destroywork(workp); 1539 return (NULL); 1540 } 1541 1542 rv = ddi_dma_mem_alloc(workp->dw_mcr_dmah, 1543 ROUNDUP(MCR_SIZE, dca->dca_pagesize), 1544 &dca_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, 1545 &workp->dw_mcr_kaddr, &size, &workp->dw_mcr_acch); 1546 if (rv != 0) { 1547 dca_error(dca, "unable to alloc MCR DMA memory"); 1548 dca_destroywork(workp); 1549 return (NULL); 1550 } 1551 1552 rv = ddi_dma_addr_bind_handle(workp->dw_mcr_dmah, NULL, 1553 workp->dw_mcr_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_RDWR, 1554 DDI_DMA_SLEEP, NULL, &c, &nc); 1555 if (rv != DDI_DMA_MAPPED) { 1556 dca_error(dca, "unable to map MCR DMA memory"); 1557 dca_destroywork(workp); 1558 return (NULL); 1559 } 1560 1561 workp->dw_mcr_paddr = c.dmac_address; 1562 return (workp); 1563 } 1564 1565 void 1566 dca_destroywork(dca_work_t *workp) 1567 { 1568 if (workp->dw_mcr_paddr) { 1569 (void) ddi_dma_unbind_handle(workp->dw_mcr_dmah); 1570 } 1571 if (workp->dw_mcr_acch) { 1572 ddi_dma_mem_free(&workp->dw_mcr_acch); 1573 } 1574 if (workp->dw_mcr_dmah) { 1575 ddi_dma_free_handle(&workp->dw_mcr_dmah); 1576 } 1577 kmem_free(workp, sizeof (dca_work_t)); 1578 } 1579 1580 dca_request_t * 1581 dca_newreq(dca_t *dca) 1582 { 1583 dca_request_t *reqp; 1584 size_t size; 1585 ddi_dma_cookie_t c; 1586 unsigned nc; 1587 int rv; 1588 int n_chain = 0; 1589 1590 size = (DESC_SIZE * MAXFRAGS) + CTX_MAXLENGTH; 1591 1592 reqp = kmem_zalloc(sizeof (dca_request_t), KM_SLEEP); 1593 1594 reqp->dr_dca = dca; 1595 1596 /* 1597 * Setup the DMA region for the context and descriptors. 1598 */ 1599 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, DDI_DMA_SLEEP, 1600 NULL, &reqp->dr_ctx_dmah); 1601 if (rv != DDI_SUCCESS) { 1602 dca_error(dca, "failure allocating request DMA handle"); 1603 dca_destroyreq(reqp); 1604 return (NULL); 1605 } 1606 1607 /* for driver hardening, allocate in whole pages */ 1608 rv = ddi_dma_mem_alloc(reqp->dr_ctx_dmah, 1609 ROUNDUP(size, dca->dca_pagesize), &dca_devattr, DDI_DMA_CONSISTENT, 1610 DDI_DMA_SLEEP, NULL, &reqp->dr_ctx_kaddr, &size, 1611 &reqp->dr_ctx_acch); 1612 if (rv != DDI_SUCCESS) { 1613 dca_error(dca, "unable to alloc request DMA memory"); 1614 dca_destroyreq(reqp); 1615 return (NULL); 1616 } 1617 1618 rv = ddi_dma_addr_bind_handle(reqp->dr_ctx_dmah, NULL, 1619 reqp->dr_ctx_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_WRITE, 1620 DDI_DMA_SLEEP, 0, &c, &nc); 1621 if (rv != DDI_DMA_MAPPED) { 1622 dca_error(dca, "failed binding request DMA handle"); 1623 dca_destroyreq(reqp); 1624 return (NULL); 1625 } 1626 reqp->dr_ctx_paddr = c.dmac_address; 1627 1628 reqp->dr_dma_size = size; 1629 1630 /* 1631 * Set up the dma for our scratch/shared buffers. 1632 */ 1633 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, 1634 DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_dmah); 1635 if (rv != DDI_SUCCESS) { 1636 dca_error(dca, "failure allocating ibuf DMA handle"); 1637 dca_destroyreq(reqp); 1638 return (NULL); 1639 } 1640 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, 1641 DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_dmah); 1642 if (rv != DDI_SUCCESS) { 1643 dca_error(dca, "failure allocating obuf DMA handle"); 1644 dca_destroyreq(reqp); 1645 return (NULL); 1646 } 1647 1648 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, 1649 DDI_DMA_SLEEP, NULL, &reqp->dr_chain_in_dmah); 1650 if (rv != DDI_SUCCESS) { 1651 dca_error(dca, "failure allocating chain_in DMA handle"); 1652 dca_destroyreq(reqp); 1653 return (NULL); 1654 } 1655 1656 rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, 1657 DDI_DMA_SLEEP, NULL, &reqp->dr_chain_out_dmah); 1658 if (rv != DDI_SUCCESS) { 1659 dca_error(dca, "failure allocating chain_out DMA handle"); 1660 dca_destroyreq(reqp); 1661 return (NULL); 1662 } 1663 1664 /* 1665 * for driver hardening, allocate in whole pages. 1666 */ 1667 size = ROUNDUP(MAXPACKET, dca->dca_pagesize); 1668 /* 1669 * We could kmem_alloc for Sparc too. However, it gives worse 1670 * performance when transferring more than one page data. For example, 1671 * using 4 threads and 12032 byte data and 3DES on 900MHZ Sparc system, 1672 * kmem_alloc uses 80% CPU and ddi_dma_mem_alloc uses 50% CPU for 1673 * the same throughput. 1674 */ 1675 rv = ddi_dma_mem_alloc(reqp->dr_ibuf_dmah, 1676 size, &dca_bufattr, 1677 DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_kaddr, 1678 &size, &reqp->dr_ibuf_acch); 1679 if (rv != DDI_SUCCESS) { 1680 dca_error(dca, "unable to alloc request DMA memory"); 1681 dca_destroyreq(reqp); 1682 return (NULL); 1683 } 1684 1685 rv = ddi_dma_mem_alloc(reqp->dr_obuf_dmah, 1686 size, &dca_bufattr, 1687 DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_kaddr, 1688 &size, &reqp->dr_obuf_acch); 1689 if (rv != DDI_SUCCESS) { 1690 dca_error(dca, "unable to alloc request DMA memory"); 1691 dca_destroyreq(reqp); 1692 return (NULL); 1693 } 1694 1695 /* Skip the used portion in the context page */ 1696 reqp->dr_offset = CTX_MAXLENGTH; 1697 if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset, 1698 reqp->dr_ibuf_kaddr, reqp->dr_ibuf_dmah, 1699 DDI_DMA_WRITE | DDI_DMA_STREAMING, 1700 &reqp->dr_ibuf_head, &n_chain)) != DDI_SUCCESS) { 1701 (void) dca_destroyreq(reqp); 1702 return (NULL); 1703 } 1704 reqp->dr_ibuf_paddr = reqp->dr_ibuf_head.dc_buffer_paddr; 1705 /* Skip the space used by the input buffer */ 1706 reqp->dr_offset += DESC_SIZE * n_chain; 1707 1708 if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset, 1709 reqp->dr_obuf_kaddr, reqp->dr_obuf_dmah, 1710 DDI_DMA_READ | DDI_DMA_STREAMING, 1711 &reqp->dr_obuf_head, &n_chain)) != DDI_SUCCESS) { 1712 (void) dca_destroyreq(reqp); 1713 return (NULL); 1714 } 1715 reqp->dr_obuf_paddr = reqp->dr_obuf_head.dc_buffer_paddr; 1716 /* Skip the space used by the output buffer */ 1717 reqp->dr_offset += DESC_SIZE * n_chain; 1718 1719 DBG(dca, DCHATTY, "CTX is 0x%p, phys 0x%x, len %d", 1720 reqp->dr_ctx_kaddr, reqp->dr_ctx_paddr, CTX_MAXLENGTH); 1721 return (reqp); 1722 } 1723 1724 void 1725 dca_destroyreq(dca_request_t *reqp) 1726 { 1727 1728 /* 1729 * Clean up DMA for the context structure. 1730 */ 1731 if (reqp->dr_ctx_paddr) { 1732 (void) ddi_dma_unbind_handle(reqp->dr_ctx_dmah); 1733 } 1734 1735 if (reqp->dr_ctx_acch) { 1736 ddi_dma_mem_free(&reqp->dr_ctx_acch); 1737 } 1738 1739 if (reqp->dr_ctx_dmah) { 1740 ddi_dma_free_handle(&reqp->dr_ctx_dmah); 1741 } 1742 1743 /* 1744 * Clean up DMA for the scratch buffer. 1745 */ 1746 if (reqp->dr_ibuf_paddr) { 1747 (void) ddi_dma_unbind_handle(reqp->dr_ibuf_dmah); 1748 } 1749 if (reqp->dr_obuf_paddr) { 1750 (void) ddi_dma_unbind_handle(reqp->dr_obuf_dmah); 1751 } 1752 1753 if (reqp->dr_ibuf_acch) { 1754 ddi_dma_mem_free(&reqp->dr_ibuf_acch); 1755 } 1756 if (reqp->dr_obuf_acch) { 1757 ddi_dma_mem_free(&reqp->dr_obuf_acch); 1758 } 1759 1760 if (reqp->dr_ibuf_dmah) { 1761 ddi_dma_free_handle(&reqp->dr_ibuf_dmah); 1762 } 1763 if (reqp->dr_obuf_dmah) { 1764 ddi_dma_free_handle(&reqp->dr_obuf_dmah); 1765 } 1766 /* 1767 * These two DMA handles should have been unbinded in 1768 * dca_unbindchains() function 1769 */ 1770 if (reqp->dr_chain_in_dmah) { 1771 ddi_dma_free_handle(&reqp->dr_chain_in_dmah); 1772 } 1773 if (reqp->dr_chain_out_dmah) { 1774 ddi_dma_free_handle(&reqp->dr_chain_out_dmah); 1775 } 1776 1777 kmem_free(reqp, sizeof (dca_request_t)); 1778 } 1779 1780 dca_work_t * 1781 dca_getwork(dca_t *dca, int mcr) 1782 { 1783 dca_worklist_t *wlp = WORKLIST(dca, mcr); 1784 dca_work_t *workp; 1785 1786 mutex_enter(&wlp->dwl_freelock); 1787 workp = (dca_work_t *)dca_dequeue(&wlp->dwl_freework); 1788 mutex_exit(&wlp->dwl_freelock); 1789 if (workp) { 1790 int nreqs; 1791 bzero(workp->dw_mcr_kaddr, 8); 1792 1793 /* clear out old requests */ 1794 for (nreqs = 0; nreqs < MAXREQSPERMCR; nreqs++) { 1795 workp->dw_reqs[nreqs] = NULL; 1796 } 1797 } 1798 return (workp); 1799 } 1800 1801 void 1802 dca_freework(dca_work_t *workp) 1803 { 1804 mutex_enter(&workp->dw_wlp->dwl_freelock); 1805 dca_enqueue(&workp->dw_wlp->dwl_freework, (dca_listnode_t *)workp); 1806 mutex_exit(&workp->dw_wlp->dwl_freelock); 1807 } 1808 1809 dca_request_t * 1810 dca_getreq(dca_t *dca, int mcr, int tryhard) 1811 { 1812 dca_worklist_t *wlp = WORKLIST(dca, mcr); 1813 dca_request_t *reqp; 1814 1815 mutex_enter(&wlp->dwl_freereqslock); 1816 reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_freereqs); 1817 mutex_exit(&wlp->dwl_freereqslock); 1818 if (reqp) { 1819 reqp->dr_flags = 0; 1820 reqp->dr_callback = NULL; 1821 } else if (tryhard) { 1822 /* 1823 * failed to get a free one, try an allocation, the hard way. 1824 * XXX: Kstat desired here. 1825 */ 1826 if ((reqp = dca_newreq(dca)) != NULL) { 1827 reqp->dr_wlp = wlp; 1828 reqp->dr_dca = dca; 1829 reqp->dr_flags = 0; 1830 reqp->dr_callback = NULL; 1831 } 1832 } 1833 return (reqp); 1834 } 1835 1836 void 1837 dca_freereq(dca_request_t *reqp) 1838 { 1839 reqp->dr_kcf_req = NULL; 1840 if (!(reqp->dr_flags & DR_NOCACHE)) { 1841 mutex_enter(&reqp->dr_wlp->dwl_freereqslock); 1842 dca_enqueue(&reqp->dr_wlp->dwl_freereqs, 1843 (dca_listnode_t *)reqp); 1844 mutex_exit(&reqp->dr_wlp->dwl_freereqslock); 1845 } 1846 } 1847 1848 /* 1849 * Binds user buffers to DMA handles dynamically. On Sparc, a user buffer 1850 * is mapped to a single physical address. On x86, a user buffer is mapped 1851 * to multiple physical addresses. These physical addresses are chained 1852 * using the method specified in Broadcom BCM5820 specification. 1853 */ 1854 int 1855 dca_bindchains(dca_request_t *reqp, size_t incnt, size_t outcnt) 1856 { 1857 int rv; 1858 caddr_t kaddr; 1859 uint_t flags; 1860 int n_chain = 0; 1861 1862 if (reqp->dr_flags & DR_INPLACE) { 1863 flags = DDI_DMA_RDWR | DDI_DMA_CONSISTENT; 1864 } else { 1865 flags = DDI_DMA_WRITE | DDI_DMA_STREAMING; 1866 } 1867 1868 /* first the input */ 1869 if (incnt) { 1870 if ((kaddr = dca_bufdaddr(reqp->dr_in)) == NULL) { 1871 DBG(NULL, DWARN, "unrecognised crypto data format"); 1872 return (DDI_FAILURE); 1873 } 1874 if ((rv = dca_bindchains_one(reqp, incnt, reqp->dr_offset, 1875 kaddr, reqp->dr_chain_in_dmah, flags, 1876 &reqp->dr_chain_in_head, &n_chain)) != DDI_SUCCESS) { 1877 (void) dca_unbindchains(reqp); 1878 return (rv); 1879 } 1880 1881 /* 1882 * The offset and length are altered by the calling routine 1883 * reqp->dr_in->cd_offset += incnt; 1884 * reqp->dr_in->cd_length -= incnt; 1885 */ 1886 /* Save the first one in the chain for MCR */ 1887 reqp->dr_in_paddr = reqp->dr_chain_in_head.dc_buffer_paddr; 1888 reqp->dr_in_next = reqp->dr_chain_in_head.dc_next_paddr; 1889 reqp->dr_in_len = reqp->dr_chain_in_head.dc_buffer_length; 1890 } else { 1891 reqp->dr_in_paddr = 0; 1892 reqp->dr_in_next = 0; 1893 reqp->dr_in_len = 0; 1894 } 1895 1896 if (reqp->dr_flags & DR_INPLACE) { 1897 reqp->dr_out_paddr = reqp->dr_in_paddr; 1898 reqp->dr_out_len = reqp->dr_in_len; 1899 reqp->dr_out_next = reqp->dr_in_next; 1900 return (DDI_SUCCESS); 1901 } 1902 1903 /* then the output */ 1904 if (outcnt) { 1905 flags = DDI_DMA_READ | DDI_DMA_STREAMING; 1906 if ((kaddr = dca_bufdaddr_out(reqp->dr_out)) == NULL) { 1907 DBG(NULL, DWARN, "unrecognised crypto data format"); 1908 (void) dca_unbindchains(reqp); 1909 return (DDI_FAILURE); 1910 } 1911 rv = dca_bindchains_one(reqp, outcnt, reqp->dr_offset + 1912 n_chain * DESC_SIZE, kaddr, reqp->dr_chain_out_dmah, 1913 flags, &reqp->dr_chain_out_head, &n_chain); 1914 if (rv != DDI_SUCCESS) { 1915 (void) dca_unbindchains(reqp); 1916 return (DDI_FAILURE); 1917 } 1918 1919 /* Save the first one in the chain for MCR */ 1920 reqp->dr_out_paddr = reqp->dr_chain_out_head.dc_buffer_paddr; 1921 reqp->dr_out_next = reqp->dr_chain_out_head.dc_next_paddr; 1922 reqp->dr_out_len = reqp->dr_chain_out_head.dc_buffer_length; 1923 } else { 1924 reqp->dr_out_paddr = 0; 1925 reqp->dr_out_next = 0; 1926 reqp->dr_out_len = 0; 1927 } 1928 1929 return (DDI_SUCCESS); 1930 } 1931 1932 /* 1933 * Unbind the user buffers from the DMA handles. 1934 */ 1935 int 1936 dca_unbindchains(dca_request_t *reqp) 1937 { 1938 int rv = DDI_SUCCESS; 1939 int rv1 = DDI_SUCCESS; 1940 1941 /* Clear the input chain */ 1942 if (reqp->dr_chain_in_head.dc_buffer_paddr != 0) { 1943 (void) ddi_dma_unbind_handle(reqp->dr_chain_in_dmah); 1944 reqp->dr_chain_in_head.dc_buffer_paddr = 0; 1945 } 1946 1947 if (reqp->dr_flags & DR_INPLACE) { 1948 return (rv); 1949 } 1950 1951 /* Clear the output chain */ 1952 if (reqp->dr_chain_out_head.dc_buffer_paddr != 0) { 1953 (void) ddi_dma_unbind_handle(reqp->dr_chain_out_dmah); 1954 reqp->dr_chain_out_head.dc_buffer_paddr = 0; 1955 } 1956 1957 return ((rv != DDI_SUCCESS)? rv : rv1); 1958 } 1959 1960 /* 1961 * Build either input chain or output chain. It is single-item chain for Sparc, 1962 * and possible mutiple-item chain for x86. 1963 */ 1964 static int 1965 dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset, 1966 caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags, 1967 dca_chain_t *head, int *n_chain) 1968 { 1969 ddi_dma_cookie_t c; 1970 uint_t nc; 1971 int rv; 1972 caddr_t chain_kaddr_pre; 1973 caddr_t chain_kaddr; 1974 uint32_t chain_paddr; 1975 int i; 1976 1977 /* Advance past the context structure to the starting address */ 1978 chain_paddr = reqp->dr_ctx_paddr + dr_offset; 1979 chain_kaddr = reqp->dr_ctx_kaddr + dr_offset; 1980 1981 /* 1982 * Bind the kernel address to the DMA handle. On x86, the actual 1983 * buffer is mapped into multiple physical addresses. On Sparc, 1984 * the actual buffer is mapped into a single address. 1985 */ 1986 rv = ddi_dma_addr_bind_handle(handle, 1987 NULL, kaddr, cnt, flags, DDI_DMA_DONTWAIT, NULL, &c, &nc); 1988 if (rv != DDI_DMA_MAPPED) { 1989 return (DDI_FAILURE); 1990 } 1991 1992 (void) ddi_dma_sync(handle, 0, cnt, DDI_DMA_SYNC_FORDEV); 1993 if ((rv = dca_check_dma_handle(reqp->dr_dca, handle, 1994 DCA_FM_ECLASS_NONE)) != DDI_SUCCESS) { 1995 reqp->destroy = TRUE; 1996 return (rv); 1997 } 1998 1999 *n_chain = nc; 2000 2001 /* Setup the data buffer chain for DMA transfer */ 2002 chain_kaddr_pre = NULL; 2003 head->dc_buffer_paddr = 0; 2004 head->dc_next_paddr = 0; 2005 head->dc_buffer_length = 0; 2006 for (i = 0; i < nc; i++) { 2007 /* PIO */ 2008 PUTDESC32(reqp, chain_kaddr, DESC_BUFADDR, c.dmac_address); 2009 PUTDESC16(reqp, chain_kaddr, DESC_RSVD, 0); 2010 PUTDESC16(reqp, chain_kaddr, DESC_LENGTH, c.dmac_size); 2011 2012 /* Remember the head of the chain */ 2013 if (head->dc_buffer_paddr == 0) { 2014 head->dc_buffer_paddr = c.dmac_address; 2015 head->dc_buffer_length = c.dmac_size; 2016 } 2017 2018 /* Link to the previous one if one exists */ 2019 if (chain_kaddr_pre) { 2020 PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT, 2021 chain_paddr); 2022 if (head->dc_next_paddr == 0) 2023 head->dc_next_paddr = chain_paddr; 2024 } 2025 chain_kaddr_pre = chain_kaddr; 2026 2027 /* Maintain pointers */ 2028 chain_paddr += DESC_SIZE; 2029 chain_kaddr += DESC_SIZE; 2030 2031 /* Retrieve the next cookie if there is one */ 2032 if (i < nc-1) 2033 ddi_dma_nextcookie(handle, &c); 2034 } 2035 2036 /* Set the next pointer in the last entry to NULL */ 2037 PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT, 0); 2038 2039 return (DDI_SUCCESS); 2040 } 2041 2042 /* 2043 * Schedule some work. 2044 */ 2045 int 2046 dca_start(dca_t *dca, dca_request_t *reqp, int mcr, int dosched) 2047 { 2048 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2049 2050 mutex_enter(&wlp->dwl_lock); 2051 2052 DBG(dca, DCHATTY, "req=%p, in=%p, out=%p, ctx=%p, ibuf=%p, obuf=%p", 2053 reqp, reqp->dr_in, reqp->dr_out, reqp->dr_ctx_kaddr, 2054 reqp->dr_ibuf_kaddr, reqp->dr_obuf_kaddr); 2055 DBG(dca, DCHATTY, "ctx paddr = %x, ibuf paddr = %x, obuf paddr = %x", 2056 reqp->dr_ctx_paddr, reqp->dr_ibuf_paddr, reqp->dr_obuf_paddr); 2057 /* sync out the entire context and descriptor chains */ 2058 (void) ddi_dma_sync(reqp->dr_ctx_dmah, 0, 0, DDI_DMA_SYNC_FORDEV); 2059 if (dca_check_dma_handle(dca, reqp->dr_ctx_dmah, 2060 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 2061 reqp->destroy = TRUE; 2062 mutex_exit(&wlp->dwl_lock); 2063 return (CRYPTO_DEVICE_ERROR); 2064 } 2065 2066 dca_enqueue(&wlp->dwl_waitq, (dca_listnode_t *)reqp); 2067 wlp->dwl_count++; 2068 wlp->dwl_lastsubmit = ddi_get_lbolt(); 2069 reqp->dr_wlp = wlp; 2070 2071 if ((wlp->dwl_count == wlp->dwl_hiwater) && (wlp->dwl_busy == 0)) { 2072 /* we are fully loaded now, let kCF know */ 2073 2074 wlp->dwl_flowctl++; 2075 wlp->dwl_busy = 1; 2076 2077 crypto_prov_notify(wlp->dwl_prov, CRYPTO_PROVIDER_BUSY); 2078 } 2079 2080 if (dosched) { 2081 #ifdef SCHEDDELAY 2082 /* possibly wait for more work to arrive */ 2083 if (wlp->dwl_count >= wlp->dwl_reqspermcr) { 2084 dca_schedule(dca, mcr); 2085 } else if (!wlp->dwl_schedtid) { 2086 /* wait 1 msec for more work before doing it */ 2087 wlp->dwl_schedtid = timeout(dca_schedtimeout, 2088 (void *)wlp, drv_usectohz(MSEC)); 2089 } 2090 #else 2091 dca_schedule(dca, mcr); 2092 #endif 2093 } 2094 mutex_exit(&wlp->dwl_lock); 2095 2096 return (CRYPTO_QUEUED); 2097 } 2098 2099 void 2100 dca_schedule(dca_t *dca, int mcr) 2101 { 2102 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2103 int csr; 2104 int full; 2105 uint32_t status; 2106 2107 ASSERT(mutex_owned(&wlp->dwl_lock)); 2108 /* 2109 * If the card is draining or has an outstanding failure, 2110 * don't schedule any more work on it right now 2111 */ 2112 if (wlp->dwl_drain || (dca->dca_flags & DCA_FAILED)) { 2113 return; 2114 } 2115 2116 if (mcr == MCR2) { 2117 csr = CSR_MCR2; 2118 full = DMASTAT_MCR2FULL; 2119 } else { 2120 csr = CSR_MCR1; 2121 full = DMASTAT_MCR1FULL; 2122 } 2123 2124 for (;;) { 2125 dca_work_t *workp; 2126 uint32_t offset; 2127 int nreqs; 2128 2129 status = GETCSR(dca, CSR_DMASTAT); 2130 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 2131 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) 2132 return; 2133 2134 if ((status & full) != 0) 2135 break; 2136 2137 #ifdef SCHEDDELAY 2138 /* if there isn't enough to do, don't bother now */ 2139 if ((wlp->dwl_count < wlp->dwl_reqspermcr) && 2140 (ddi_get_lbolt() < (wlp->dwl_lastsubmit + 2141 drv_usectohz(MSEC)))) { 2142 /* wait a bit longer... */ 2143 if (wlp->dwl_schedtid == 0) { 2144 wlp->dwl_schedtid = timeout(dca_schedtimeout, 2145 (void *)wlp, drv_usectohz(MSEC)); 2146 } 2147 return; 2148 } 2149 #endif 2150 2151 /* grab a work structure */ 2152 workp = dca_getwork(dca, mcr); 2153 2154 if (workp == NULL) { 2155 /* 2156 * There must be work ready to be reclaimed, 2157 * in this case, since the chip can only hold 2158 * less work outstanding than there are total. 2159 */ 2160 dca_reclaim(dca, mcr); 2161 continue; 2162 } 2163 2164 nreqs = 0; 2165 offset = MCR_CTXADDR; 2166 2167 while (nreqs < wlp->dwl_reqspermcr) { 2168 dca_request_t *reqp; 2169 2170 reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_waitq); 2171 if (reqp == NULL) { 2172 /* nothing left to process */ 2173 break; 2174 } 2175 /* 2176 * Update flow control. 2177 */ 2178 wlp->dwl_count--; 2179 if ((wlp->dwl_count == wlp->dwl_lowater) && 2180 (wlp->dwl_busy)) { 2181 wlp->dwl_busy = 0; 2182 crypto_prov_notify(wlp->dwl_prov, 2183 CRYPTO_PROVIDER_READY); 2184 } 2185 2186 /* 2187 * Context address. 2188 */ 2189 PUTMCR32(workp, offset, reqp->dr_ctx_paddr); 2190 offset += 4; 2191 2192 /* 2193 * Input chain. 2194 */ 2195 /* input buffer address */ 2196 PUTMCR32(workp, offset, reqp->dr_in_paddr); 2197 offset += 4; 2198 /* next input buffer entry */ 2199 PUTMCR32(workp, offset, reqp->dr_in_next); 2200 offset += 4; 2201 /* input buffer length */ 2202 PUTMCR16(workp, offset, reqp->dr_in_len); 2203 offset += 2; 2204 /* zero the reserved field */ 2205 PUTMCR16(workp, offset, 0); 2206 offset += 2; 2207 2208 /* 2209 * Overall length. 2210 */ 2211 /* reserved field */ 2212 PUTMCR16(workp, offset, 0); 2213 offset += 2; 2214 /* total packet length */ 2215 PUTMCR16(workp, offset, reqp->dr_pkt_length); 2216 offset += 2; 2217 2218 /* 2219 * Output chain. 2220 */ 2221 /* output buffer address */ 2222 PUTMCR32(workp, offset, reqp->dr_out_paddr); 2223 offset += 4; 2224 /* next output buffer entry */ 2225 PUTMCR32(workp, offset, reqp->dr_out_next); 2226 offset += 4; 2227 /* output buffer length */ 2228 PUTMCR16(workp, offset, reqp->dr_out_len); 2229 offset += 2; 2230 /* zero the reserved field */ 2231 PUTMCR16(workp, offset, 0); 2232 offset += 2; 2233 2234 /* 2235 * Note submission. 2236 */ 2237 workp->dw_reqs[nreqs] = reqp; 2238 nreqs++; 2239 } 2240 2241 if (nreqs == 0) { 2242 /* nothing in the queue! */ 2243 dca_freework(workp); 2244 return; 2245 } 2246 2247 wlp->dwl_submit++; 2248 2249 PUTMCR16(workp, MCR_FLAGS, 0); 2250 PUTMCR16(workp, MCR_COUNT, nreqs); 2251 2252 DBG(dca, DCHATTY, 2253 "posting work (phys %x, virt 0x%p) (%d reqs) to MCR%d", 2254 workp->dw_mcr_paddr, workp->dw_mcr_kaddr, 2255 nreqs, mcr); 2256 2257 workp->dw_lbolt = ddi_get_lbolt(); 2258 /* Make sure MCR is synced out to device. */ 2259 (void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 0, 2260 DDI_DMA_SYNC_FORDEV); 2261 if (dca_check_dma_handle(dca, workp->dw_mcr_dmah, 2262 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 2263 dca_destroywork(workp); 2264 return; 2265 } 2266 2267 PUTCSR(dca, csr, workp->dw_mcr_paddr); 2268 if (dca_check_acc_handle(dca, dca->dca_regs_handle, 2269 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 2270 dca_destroywork(workp); 2271 return; 2272 } else { 2273 dca_enqueue(&wlp->dwl_runq, (dca_listnode_t *)workp); 2274 } 2275 2276 DBG(dca, DCHATTY, "posted"); 2277 } 2278 } 2279 2280 /* 2281 * Reclaim completed work, called in interrupt context. 2282 */ 2283 void 2284 dca_reclaim(dca_t *dca, int mcr) 2285 { 2286 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2287 dca_work_t *workp; 2288 ushort_t flags; 2289 int nreclaimed = 0; 2290 int i; 2291 2292 DBG(dca, DRECLAIM, "worklist = 0x%p (MCR%d)", wlp, mcr); 2293 ASSERT(mutex_owned(&wlp->dwl_lock)); 2294 /* 2295 * For each MCR in the submitted (runq), we check to see if 2296 * it has been processed. If so, then we note each individual 2297 * job in the MCR, and and do the completion processing for 2298 * each of such job. 2299 */ 2300 for (;;) { 2301 2302 workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq); 2303 if (workp == NULL) { 2304 break; 2305 } 2306 2307 /* only sync the MCR flags, since that's all we need */ 2308 (void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 4, 2309 DDI_DMA_SYNC_FORKERNEL); 2310 if (dca_check_dma_handle(dca, workp->dw_mcr_dmah, 2311 DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { 2312 dca_rmqueue((dca_listnode_t *)workp); 2313 dca_destroywork(workp); 2314 return; 2315 } 2316 2317 flags = GETMCR16(workp, MCR_FLAGS); 2318 if ((flags & MCRFLAG_FINISHED) == 0) { 2319 /* chip is still working on it */ 2320 DBG(dca, DRECLAIM, 2321 "chip still working on it (MCR%d)", mcr); 2322 break; 2323 } 2324 2325 /* its really for us, so remove it from the queue */ 2326 dca_rmqueue((dca_listnode_t *)workp); 2327 2328 /* if we were draining, signal on the cv */ 2329 if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) { 2330 cv_signal(&wlp->dwl_cv); 2331 } 2332 2333 /* update statistics, done under the lock */ 2334 for (i = 0; i < wlp->dwl_reqspermcr; i++) { 2335 dca_request_t *reqp = workp->dw_reqs[i]; 2336 if (reqp == NULL) { 2337 continue; 2338 } 2339 if (reqp->dr_byte_stat >= 0) { 2340 dca->dca_stats[reqp->dr_byte_stat] += 2341 reqp->dr_pkt_length; 2342 } 2343 if (reqp->dr_job_stat >= 0) { 2344 dca->dca_stats[reqp->dr_job_stat]++; 2345 } 2346 } 2347 mutex_exit(&wlp->dwl_lock); 2348 2349 for (i = 0; i < wlp->dwl_reqspermcr; i++) { 2350 dca_request_t *reqp = workp->dw_reqs[i]; 2351 2352 if (reqp == NULL) { 2353 continue; 2354 } 2355 2356 /* Do the callback. */ 2357 workp->dw_reqs[i] = NULL; 2358 dca_done(reqp, CRYPTO_SUCCESS); 2359 2360 nreclaimed++; 2361 } 2362 2363 /* now we can release the work */ 2364 dca_freework(workp); 2365 2366 mutex_enter(&wlp->dwl_lock); 2367 } 2368 DBG(dca, DRECLAIM, "reclaimed %d cmds", nreclaimed); 2369 } 2370 2371 int 2372 dca_length(crypto_data_t *cdata) 2373 { 2374 return (cdata->cd_length); 2375 } 2376 2377 /* 2378 * This is the callback function called from the interrupt when a kCF job 2379 * completes. It does some driver-specific things, and then calls the 2380 * kCF-provided callback. Finally, it cleans up the state for the work 2381 * request and drops the reference count to allow for DR. 2382 */ 2383 void 2384 dca_done(dca_request_t *reqp, int err) 2385 { 2386 uint64_t ena = 0; 2387 2388 /* unbind any chains we were using */ 2389 if (dca_unbindchains(reqp) != DDI_SUCCESS) { 2390 /* DMA failure */ 2391 ena = dca_ena(ena); 2392 dca_failure(reqp->dr_dca, DDI_DATAPATH_FAULT, 2393 DCA_FM_ECLASS_NONE, ena, CRYPTO_DEVICE_ERROR, 2394 "fault on buffer DMA handle"); 2395 if (err == CRYPTO_SUCCESS) { 2396 err = CRYPTO_DEVICE_ERROR; 2397 } 2398 } 2399 2400 if (reqp->dr_callback != NULL) { 2401 reqp->dr_callback(reqp, err); 2402 } else { 2403 dca_freereq(reqp); 2404 } 2405 } 2406 2407 /* 2408 * Call this when a failure is detected. It will reset the chip, 2409 * log a message, alert kCF, and mark jobs in the runq as failed. 2410 */ 2411 /* ARGSUSED */ 2412 void 2413 dca_failure(dca_t *dca, ddi_fault_location_t loc, dca_fma_eclass_t index, 2414 uint64_t ena, int errno, char *mess, ...) 2415 { 2416 va_list ap; 2417 char buf[256]; 2418 int mcr; 2419 char *eclass; 2420 int have_mutex; 2421 2422 va_start(ap, mess); 2423 (void) vsprintf(buf, mess, ap); 2424 va_end(ap); 2425 2426 eclass = dca_fma_eclass_string(dca->dca_model, index); 2427 2428 if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) && 2429 index != DCA_FM_ECLASS_NONE) { 2430 ddi_fm_ereport_post(dca->dca_dip, eclass, ena, 2431 DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8, 2432 FM_EREPORT_VERS0, NULL); 2433 2434 /* Report the impact of the failure to the DDI. */ 2435 ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_LOST); 2436 } else { 2437 /* Just log the error string to the message log */ 2438 dca_error(dca, buf); 2439 } 2440 2441 /* 2442 * Indicate a failure (keeps schedule from running). 2443 */ 2444 dca->dca_flags |= DCA_FAILED; 2445 2446 /* 2447 * Reset the chip. This should also have as a side effect, the 2448 * disabling of all interrupts from the device. 2449 */ 2450 (void) dca_reset(dca, 1); 2451 2452 /* 2453 * Report the failure to kCF. 2454 */ 2455 for (mcr = MCR1; mcr <= MCR2; mcr++) { 2456 if (WORKLIST(dca, mcr)->dwl_prov) { 2457 crypto_prov_notify(WORKLIST(dca, mcr)->dwl_prov, 2458 CRYPTO_PROVIDER_FAILED); 2459 } 2460 } 2461 2462 /* 2463 * Return jobs not sent to hardware back to kCF. 2464 */ 2465 dca_rejectjobs(dca); 2466 2467 /* 2468 * From this point on, no new work should be arriving, and the 2469 * chip should not be doing any active DMA. 2470 */ 2471 2472 /* 2473 * Now find all the work submitted to the device and fail 2474 * them. 2475 */ 2476 for (mcr = MCR1; mcr <= MCR2; mcr++) { 2477 dca_worklist_t *wlp; 2478 int i; 2479 2480 wlp = WORKLIST(dca, mcr); 2481 2482 if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) { 2483 continue; 2484 } 2485 for (;;) { 2486 dca_work_t *workp; 2487 2488 have_mutex = mutex_tryenter(&wlp->dwl_lock); 2489 workp = (dca_work_t *)dca_dequeue(&wlp->dwl_runq); 2490 if (workp == NULL) { 2491 if (have_mutex) 2492 mutex_exit(&wlp->dwl_lock); 2493 break; 2494 } 2495 mutex_exit(&wlp->dwl_lock); 2496 2497 /* 2498 * Free up requests 2499 */ 2500 for (i = 0; i < wlp->dwl_reqspermcr; i++) { 2501 dca_request_t *reqp = workp->dw_reqs[i]; 2502 if (reqp) { 2503 dca_done(reqp, errno); 2504 workp->dw_reqs[i] = NULL; 2505 } 2506 } 2507 2508 mutex_enter(&wlp->dwl_lock); 2509 /* 2510 * If waiting to drain, signal on the waiter. 2511 */ 2512 if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) { 2513 cv_signal(&wlp->dwl_cv); 2514 } 2515 2516 /* 2517 * Return the work and request structures to 2518 * the free pool. 2519 */ 2520 dca_freework(workp); 2521 if (have_mutex) 2522 mutex_exit(&wlp->dwl_lock); 2523 } 2524 } 2525 2526 } 2527 2528 #ifdef SCHEDDELAY 2529 /* 2530 * Reschedule worklist as needed. 2531 */ 2532 void 2533 dca_schedtimeout(void *arg) 2534 { 2535 dca_worklist_t *wlp = (dca_worklist_t *)arg; 2536 mutex_enter(&wlp->dwl_lock); 2537 wlp->dwl_schedtid = 0; 2538 dca_schedule(wlp->dwl_dca, wlp->dwl_mcr); 2539 mutex_exit(&wlp->dwl_lock); 2540 } 2541 #endif 2542 2543 /* 2544 * Check for stalled jobs. 2545 */ 2546 void 2547 dca_jobtimeout(void *arg) 2548 { 2549 int mcr; 2550 dca_t *dca = (dca_t *)arg; 2551 int hung = 0; 2552 2553 for (mcr = MCR1; mcr <= MCR2; mcr++) { 2554 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2555 dca_work_t *workp; 2556 clock_t when; 2557 2558 mutex_enter(&wlp->dwl_lock); 2559 when = ddi_get_lbolt(); 2560 2561 workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq); 2562 if (workp == NULL) { 2563 /* nothing sitting in the queue */ 2564 mutex_exit(&wlp->dwl_lock); 2565 continue; 2566 } 2567 2568 if ((when - workp->dw_lbolt) < drv_usectohz(STALETIME)) { 2569 /* request has been queued for less than STALETIME */ 2570 mutex_exit(&wlp->dwl_lock); 2571 continue; 2572 } 2573 2574 /* job has been sitting around for over 1 second, badness */ 2575 DBG(dca, DWARN, "stale job (0x%p) found in MCR%d!", workp, 2576 mcr); 2577 2578 /* put it back in the queue, until we reset the chip */ 2579 hung++; 2580 mutex_exit(&wlp->dwl_lock); 2581 } 2582 2583 if (hung) { 2584 dca_failure(dca, DDI_DEVICE_FAULT, 2585 DCA_FM_ECLASS_HW_TIMEOUT, dca_ena(0), CRYPTO_DEVICE_ERROR, 2586 "timeout processing job.)"); 2587 } 2588 2589 /* reschedule ourself */ 2590 mutex_enter(&dca->dca_intrlock); 2591 if (dca->dca_jobtid == 0) { 2592 /* timeout has been canceled, prior to DR */ 2593 mutex_exit(&dca->dca_intrlock); 2594 return; 2595 } 2596 2597 /* check again in 1 second */ 2598 dca->dca_jobtid = timeout(dca_jobtimeout, arg, 2599 drv_usectohz(SECOND)); 2600 mutex_exit(&dca->dca_intrlock); 2601 } 2602 2603 /* 2604 * This returns all jobs back to kCF. It assumes that processing 2605 * on the worklist has halted. 2606 */ 2607 void 2608 dca_rejectjobs(dca_t *dca) 2609 { 2610 int mcr; 2611 int have_mutex; 2612 for (mcr = MCR1; mcr <= MCR2; mcr++) { 2613 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2614 dca_request_t *reqp; 2615 2616 if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) { 2617 continue; 2618 } 2619 have_mutex = mutex_tryenter(&wlp->dwl_lock); 2620 for (;;) { 2621 reqp = (dca_request_t *)dca_unqueue(&wlp->dwl_waitq); 2622 if (reqp == NULL) { 2623 break; 2624 } 2625 /* update flow control */ 2626 wlp->dwl_count--; 2627 if ((wlp->dwl_count == wlp->dwl_lowater) && 2628 (wlp->dwl_busy)) { 2629 wlp->dwl_busy = 0; 2630 crypto_prov_notify(wlp->dwl_prov, 2631 CRYPTO_PROVIDER_READY); 2632 } 2633 mutex_exit(&wlp->dwl_lock); 2634 2635 (void) dca_unbindchains(reqp); 2636 reqp->dr_callback(reqp, EAGAIN); 2637 mutex_enter(&wlp->dwl_lock); 2638 } 2639 if (have_mutex) 2640 mutex_exit(&wlp->dwl_lock); 2641 } 2642 } 2643 2644 int 2645 dca_drain(dca_t *dca) 2646 { 2647 int mcr; 2648 for (mcr = MCR1; mcr <= MCR2; mcr++) { 2649 #ifdef SCHEDDELAY 2650 timeout_id_t tid; 2651 #endif 2652 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2653 2654 mutex_enter(&wlp->dwl_lock); 2655 wlp->dwl_drain = 1; 2656 2657 /* give it up to a second to drain from the chip */ 2658 if (!QEMPTY(&wlp->dwl_runq)) { 2659 (void) cv_reltimedwait(&wlp->dwl_cv, &wlp->dwl_lock, 2660 drv_usectohz(STALETIME), TR_CLOCK_TICK); 2661 2662 if (!QEMPTY(&wlp->dwl_runq)) { 2663 dca_error(dca, "unable to drain device"); 2664 mutex_exit(&wlp->dwl_lock); 2665 dca_undrain(dca); 2666 return (EBUSY); 2667 } 2668 } 2669 2670 #ifdef SCHEDDELAY 2671 tid = wlp->dwl_schedtid; 2672 mutex_exit(&wlp->dwl_lock); 2673 2674 /* 2675 * untimeout outside the lock -- this is safe because we 2676 * have set the drain flag, so dca_schedule() will not 2677 * reschedule another timeout 2678 */ 2679 if (tid) { 2680 untimeout(tid); 2681 } 2682 #else 2683 mutex_exit(&wlp->dwl_lock); 2684 #endif 2685 } 2686 return (0); 2687 } 2688 2689 void 2690 dca_undrain(dca_t *dca) 2691 { 2692 int mcr; 2693 2694 for (mcr = MCR1; mcr <= MCR2; mcr++) { 2695 dca_worklist_t *wlp = WORKLIST(dca, mcr); 2696 mutex_enter(&wlp->dwl_lock); 2697 wlp->dwl_drain = 0; 2698 dca_schedule(dca, mcr); 2699 mutex_exit(&wlp->dwl_lock); 2700 } 2701 } 2702 2703 /* 2704 * Duplicate the crypto_data_t structure, but point to the original 2705 * buffers. 2706 */ 2707 int 2708 dca_dupcrypto(crypto_data_t *input, crypto_data_t *ninput) 2709 { 2710 ninput->cd_format = input->cd_format; 2711 ninput->cd_offset = input->cd_offset; 2712 ninput->cd_length = input->cd_length; 2713 ninput->cd_miscdata = input->cd_miscdata; 2714 2715 switch (input->cd_format) { 2716 case CRYPTO_DATA_RAW: 2717 ninput->cd_raw.iov_base = input->cd_raw.iov_base; 2718 ninput->cd_raw.iov_len = input->cd_raw.iov_len; 2719 break; 2720 2721 case CRYPTO_DATA_UIO: 2722 ninput->cd_uio = input->cd_uio; 2723 break; 2724 2725 case CRYPTO_DATA_MBLK: 2726 ninput->cd_mp = input->cd_mp; 2727 break; 2728 2729 default: 2730 DBG(NULL, DWARN, 2731 "dca_dupcrypto: unrecognised crypto data format"); 2732 return (CRYPTO_FAILED); 2733 } 2734 2735 return (CRYPTO_SUCCESS); 2736 } 2737 2738 /* 2739 * Performs validation checks on the input and output data structures. 2740 */ 2741 int 2742 dca_verifyio(crypto_data_t *input, crypto_data_t *output) 2743 { 2744 int rv = CRYPTO_SUCCESS; 2745 2746 switch (input->cd_format) { 2747 case CRYPTO_DATA_RAW: 2748 break; 2749 2750 case CRYPTO_DATA_UIO: 2751 /* we support only kernel buffer */ 2752 if (input->cd_uio->uio_segflg != UIO_SYSSPACE) { 2753 DBG(NULL, DWARN, "non kernel input uio buffer"); 2754 rv = CRYPTO_ARGUMENTS_BAD; 2755 } 2756 break; 2757 2758 case CRYPTO_DATA_MBLK: 2759 break; 2760 2761 default: 2762 DBG(NULL, DWARN, "unrecognised input crypto data format"); 2763 rv = CRYPTO_ARGUMENTS_BAD; 2764 } 2765 2766 switch (output->cd_format) { 2767 case CRYPTO_DATA_RAW: 2768 break; 2769 2770 case CRYPTO_DATA_UIO: 2771 /* we support only kernel buffer */ 2772 if (output->cd_uio->uio_segflg != UIO_SYSSPACE) { 2773 DBG(NULL, DWARN, "non kernel output uio buffer"); 2774 rv = CRYPTO_ARGUMENTS_BAD; 2775 } 2776 break; 2777 2778 case CRYPTO_DATA_MBLK: 2779 break; 2780 2781 default: 2782 DBG(NULL, DWARN, "unrecognised output crypto data format"); 2783 rv = CRYPTO_ARGUMENTS_BAD; 2784 } 2785 2786 return (rv); 2787 } 2788 2789 /* 2790 * data: source crypto_data_t struct 2791 * off: offset into the source before commencing copy 2792 * count: the amount of data to copy 2793 * dest: destination buffer 2794 */ 2795 int 2796 dca_getbufbytes(crypto_data_t *data, size_t off, int count, uchar_t *dest) 2797 { 2798 int rv = CRYPTO_SUCCESS; 2799 uio_t *uiop; 2800 uint_t vec_idx; 2801 size_t cur_len; 2802 mblk_t *mp; 2803 2804 if (count == 0) { 2805 /* We don't want anything so we're done. */ 2806 return (rv); 2807 } 2808 2809 /* 2810 * Sanity check that we haven't specified a length greater than the 2811 * offset adjusted size of the buffer. 2812 */ 2813 if (count > (data->cd_length - off)) { 2814 return (CRYPTO_DATA_LEN_RANGE); 2815 } 2816 2817 /* Add the internal crypto_data offset to the requested offset. */ 2818 off += data->cd_offset; 2819 2820 switch (data->cd_format) { 2821 case CRYPTO_DATA_RAW: 2822 bcopy(data->cd_raw.iov_base + off, dest, count); 2823 break; 2824 2825 case CRYPTO_DATA_UIO: 2826 /* 2827 * Jump to the first iovec containing data to be 2828 * processed. 2829 */ 2830 uiop = data->cd_uio; 2831 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && 2832 off >= uiop->uio_iov[vec_idx].iov_len; 2833 off -= uiop->uio_iov[vec_idx++].iov_len) 2834 ; 2835 if (vec_idx == uiop->uio_iovcnt) { 2836 /* 2837 * The caller specified an offset that is larger than 2838 * the total size of the buffers it provided. 2839 */ 2840 return (CRYPTO_DATA_LEN_RANGE); 2841 } 2842 2843 /* 2844 * Now process the iovecs. 2845 */ 2846 while (vec_idx < uiop->uio_iovcnt && count > 0) { 2847 cur_len = min(uiop->uio_iov[vec_idx].iov_len - 2848 off, count); 2849 bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest, 2850 cur_len); 2851 count -= cur_len; 2852 dest += cur_len; 2853 vec_idx++; 2854 off = 0; 2855 } 2856 2857 if (vec_idx == uiop->uio_iovcnt && count > 0) { 2858 /* 2859 * The end of the specified iovec's was reached but 2860 * the length requested could not be processed 2861 * (requested to digest more data than it provided). 2862 */ 2863 return (CRYPTO_DATA_LEN_RANGE); 2864 } 2865 break; 2866 2867 case CRYPTO_DATA_MBLK: 2868 /* 2869 * Jump to the first mblk_t containing data to be processed. 2870 */ 2871 for (mp = data->cd_mp; mp != NULL && off >= MBLKL(mp); 2872 off -= MBLKL(mp), mp = mp->b_cont) 2873 ; 2874 if (mp == NULL) { 2875 /* 2876 * The caller specified an offset that is larger than 2877 * the total size of the buffers it provided. 2878 */ 2879 return (CRYPTO_DATA_LEN_RANGE); 2880 } 2881 2882 /* 2883 * Now do the processing on the mblk chain. 2884 */ 2885 while (mp != NULL && count > 0) { 2886 cur_len = min(MBLKL(mp) - off, count); 2887 bcopy((char *)(mp->b_rptr + off), dest, cur_len); 2888 count -= cur_len; 2889 dest += cur_len; 2890 mp = mp->b_cont; 2891 off = 0; 2892 } 2893 2894 if (mp == NULL && count > 0) { 2895 /* 2896 * The end of the mblk was reached but the length 2897 * requested could not be processed, (requested to 2898 * digest more data than it provided). 2899 */ 2900 return (CRYPTO_DATA_LEN_RANGE); 2901 } 2902 break; 2903 2904 default: 2905 DBG(NULL, DWARN, "unrecognised crypto data format"); 2906 rv = CRYPTO_ARGUMENTS_BAD; 2907 } 2908 return (rv); 2909 } 2910 2911 2912 /* 2913 * Performs the input, output or hard scatter/gather checks on the specified 2914 * crypto_data_t struct. Returns true if the data is scatter/gather in nature 2915 * ie fails the test. 2916 */ 2917 int 2918 dca_sgcheck(dca_t *dca, crypto_data_t *data, dca_sg_param_t val) 2919 { 2920 uio_t *uiop; 2921 mblk_t *mp; 2922 int rv = FALSE; 2923 2924 switch (val) { 2925 case DCA_SG_CONTIG: 2926 /* 2927 * Check for a contiguous data buffer. 2928 */ 2929 switch (data->cd_format) { 2930 case CRYPTO_DATA_RAW: 2931 /* Contiguous in nature */ 2932 break; 2933 2934 case CRYPTO_DATA_UIO: 2935 if (data->cd_uio->uio_iovcnt > 1) 2936 rv = TRUE; 2937 break; 2938 2939 case CRYPTO_DATA_MBLK: 2940 mp = data->cd_mp; 2941 if (mp->b_cont != NULL) 2942 rv = TRUE; 2943 break; 2944 2945 default: 2946 DBG(NULL, DWARN, "unrecognised crypto data format"); 2947 } 2948 break; 2949 2950 case DCA_SG_WALIGN: 2951 /* 2952 * Check for a contiguous data buffer that is 32-bit word 2953 * aligned and is of word multiples in size. 2954 */ 2955 switch (data->cd_format) { 2956 case CRYPTO_DATA_RAW: 2957 if ((data->cd_raw.iov_len % sizeof (uint32_t)) || 2958 ((uintptr_t)data->cd_raw.iov_base % 2959 sizeof (uint32_t))) { 2960 rv = TRUE; 2961 } 2962 break; 2963 2964 case CRYPTO_DATA_UIO: 2965 uiop = data->cd_uio; 2966 if (uiop->uio_iovcnt > 1) { 2967 return (TRUE); 2968 } 2969 /* So there is only one iovec */ 2970 if ((uiop->uio_iov[0].iov_len % sizeof (uint32_t)) || 2971 ((uintptr_t)uiop->uio_iov[0].iov_base % 2972 sizeof (uint32_t))) { 2973 rv = TRUE; 2974 } 2975 break; 2976 2977 case CRYPTO_DATA_MBLK: 2978 mp = data->cd_mp; 2979 if (mp->b_cont != NULL) { 2980 return (TRUE); 2981 } 2982 /* So there is only one mblk in the chain */ 2983 if ((MBLKL(mp) % sizeof (uint32_t)) || 2984 ((uintptr_t)mp->b_rptr % sizeof (uint32_t))) { 2985 rv = TRUE; 2986 } 2987 break; 2988 2989 default: 2990 DBG(NULL, DWARN, "unrecognised crypto data format"); 2991 } 2992 break; 2993 2994 case DCA_SG_PALIGN: 2995 /* 2996 * Check that the data buffer is page aligned and is of 2997 * page multiples in size. 2998 */ 2999 switch (data->cd_format) { 3000 case CRYPTO_DATA_RAW: 3001 if ((data->cd_length % dca->dca_pagesize) || 3002 ((uintptr_t)data->cd_raw.iov_base % 3003 dca->dca_pagesize)) { 3004 rv = TRUE; 3005 } 3006 break; 3007 3008 case CRYPTO_DATA_UIO: 3009 uiop = data->cd_uio; 3010 if ((uiop->uio_iov[0].iov_len % dca->dca_pagesize) || 3011 ((uintptr_t)uiop->uio_iov[0].iov_base % 3012 dca->dca_pagesize)) { 3013 rv = TRUE; 3014 } 3015 break; 3016 3017 case CRYPTO_DATA_MBLK: 3018 mp = data->cd_mp; 3019 if ((MBLKL(mp) % dca->dca_pagesize) || 3020 ((uintptr_t)mp->b_rptr % dca->dca_pagesize)) { 3021 rv = TRUE; 3022 } 3023 break; 3024 3025 default: 3026 DBG(NULL, DWARN, "unrecognised crypto data format"); 3027 } 3028 break; 3029 3030 default: 3031 DBG(NULL, DWARN, "unrecognised scatter/gather param type"); 3032 } 3033 3034 return (rv); 3035 } 3036 3037 /* 3038 * Increments the cd_offset and decrements the cd_length as the data is 3039 * gathered from the crypto_data_t struct. 3040 * The data is reverse-copied into the dest buffer if the flag is true. 3041 */ 3042 int 3043 dca_gather(crypto_data_t *in, char *dest, int count, int reverse) 3044 { 3045 int rv = CRYPTO_SUCCESS; 3046 uint_t vec_idx; 3047 uio_t *uiop; 3048 off_t off = in->cd_offset; 3049 size_t cur_len; 3050 mblk_t *mp; 3051 3052 switch (in->cd_format) { 3053 case CRYPTO_DATA_RAW: 3054 if (count > in->cd_length) { 3055 /* 3056 * The caller specified a length greater than the 3057 * size of the buffer. 3058 */ 3059 return (CRYPTO_DATA_LEN_RANGE); 3060 } 3061 if (reverse) 3062 dca_reverse(in->cd_raw.iov_base + off, dest, count, 3063 count); 3064 else 3065 bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count); 3066 in->cd_offset += count; 3067 in->cd_length -= count; 3068 break; 3069 3070 case CRYPTO_DATA_UIO: 3071 /* 3072 * Jump to the first iovec containing data to be processed. 3073 */ 3074 uiop = in->cd_uio; 3075 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && 3076 off >= uiop->uio_iov[vec_idx].iov_len; 3077 off -= uiop->uio_iov[vec_idx++].iov_len) 3078 ; 3079 if (vec_idx == uiop->uio_iovcnt) { 3080 /* 3081 * The caller specified an offset that is larger than 3082 * the total size of the buffers it provided. 3083 */ 3084 return (CRYPTO_DATA_LEN_RANGE); 3085 } 3086 3087 /* 3088 * Now process the iovecs. 3089 */ 3090 while (vec_idx < uiop->uio_iovcnt && count > 0) { 3091 cur_len = min(uiop->uio_iov[vec_idx].iov_len - 3092 off, count); 3093 count -= cur_len; 3094 if (reverse) { 3095 /* Fill the dest buffer from the end */ 3096 dca_reverse(uiop->uio_iov[vec_idx].iov_base + 3097 off, dest+count, cur_len, cur_len); 3098 } else { 3099 bcopy(uiop->uio_iov[vec_idx].iov_base + off, 3100 dest, cur_len); 3101 dest += cur_len; 3102 } 3103 in->cd_offset += cur_len; 3104 in->cd_length -= cur_len; 3105 vec_idx++; 3106 off = 0; 3107 } 3108 3109 if (vec_idx == uiop->uio_iovcnt && count > 0) { 3110 /* 3111 * The end of the specified iovec's was reached but 3112 * the length requested could not be processed 3113 * (requested to digest more data than it provided). 3114 */ 3115 return (CRYPTO_DATA_LEN_RANGE); 3116 } 3117 break; 3118 3119 case CRYPTO_DATA_MBLK: 3120 /* 3121 * Jump to the first mblk_t containing data to be processed. 3122 */ 3123 for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp); 3124 off -= MBLKL(mp), mp = mp->b_cont) 3125 ; 3126 if (mp == NULL) { 3127 /* 3128 * The caller specified an offset that is larger than 3129 * the total size of the buffers it provided. 3130 */ 3131 return (CRYPTO_DATA_LEN_RANGE); 3132 } 3133 3134 /* 3135 * Now do the processing on the mblk chain. 3136 */ 3137 while (mp != NULL && count > 0) { 3138 cur_len = min(MBLKL(mp) - off, count); 3139 count -= cur_len; 3140 if (reverse) { 3141 /* Fill the dest buffer from the end */ 3142 dca_reverse((char *)(mp->b_rptr + off), 3143 dest+count, cur_len, cur_len); 3144 } else { 3145 bcopy((char *)(mp->b_rptr + off), dest, 3146 cur_len); 3147 dest += cur_len; 3148 } 3149 in->cd_offset += cur_len; 3150 in->cd_length -= cur_len; 3151 mp = mp->b_cont; 3152 off = 0; 3153 } 3154 3155 if (mp == NULL && count > 0) { 3156 /* 3157 * The end of the mblk was reached but the length 3158 * requested could not be processed, (requested to 3159 * digest more data than it provided). 3160 */ 3161 return (CRYPTO_DATA_LEN_RANGE); 3162 } 3163 break; 3164 3165 default: 3166 DBG(NULL, DWARN, "dca_gather: unrecognised crypto data format"); 3167 rv = CRYPTO_ARGUMENTS_BAD; 3168 } 3169 return (rv); 3170 } 3171 3172 /* 3173 * Increments the cd_offset and decrements the cd_length as the data is 3174 * gathered from the crypto_data_t struct. 3175 */ 3176 int 3177 dca_resid_gather(crypto_data_t *in, char *resid, int *residlen, char *dest, 3178 int count) 3179 { 3180 int rv = CRYPTO_SUCCESS; 3181 caddr_t baddr; 3182 uint_t vec_idx; 3183 uio_t *uiop; 3184 off_t off = in->cd_offset; 3185 size_t cur_len; 3186 mblk_t *mp; 3187 3188 /* Process the residual first */ 3189 if (*residlen > 0) { 3190 uint_t num = min(count, *residlen); 3191 bcopy(resid, dest, num); 3192 *residlen -= num; 3193 if (*residlen > 0) { 3194 /* 3195 * Requested amount 'count' is less than what's in 3196 * the residual, so shuffle any remaining resid to 3197 * the front. 3198 */ 3199 baddr = resid + num; 3200 bcopy(baddr, resid, *residlen); 3201 } 3202 dest += num; 3203 count -= num; 3204 } 3205 3206 /* Now process what's in the crypto_data_t structs */ 3207 switch (in->cd_format) { 3208 case CRYPTO_DATA_RAW: 3209 if (count > in->cd_length) { 3210 /* 3211 * The caller specified a length greater than the 3212 * size of the buffer. 3213 */ 3214 return (CRYPTO_DATA_LEN_RANGE); 3215 } 3216 bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count); 3217 in->cd_offset += count; 3218 in->cd_length -= count; 3219 break; 3220 3221 case CRYPTO_DATA_UIO: 3222 /* 3223 * Jump to the first iovec containing data to be processed. 3224 */ 3225 uiop = in->cd_uio; 3226 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && 3227 off >= uiop->uio_iov[vec_idx].iov_len; 3228 off -= uiop->uio_iov[vec_idx++].iov_len) 3229 ; 3230 if (vec_idx == uiop->uio_iovcnt) { 3231 /* 3232 * The caller specified an offset that is larger than 3233 * the total size of the buffers it provided. 3234 */ 3235 return (CRYPTO_DATA_LEN_RANGE); 3236 } 3237 3238 /* 3239 * Now process the iovecs. 3240 */ 3241 while (vec_idx < uiop->uio_iovcnt && count > 0) { 3242 cur_len = min(uiop->uio_iov[vec_idx].iov_len - 3243 off, count); 3244 bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest, 3245 cur_len); 3246 count -= cur_len; 3247 dest += cur_len; 3248 in->cd_offset += cur_len; 3249 in->cd_length -= cur_len; 3250 vec_idx++; 3251 off = 0; 3252 } 3253 3254 if (vec_idx == uiop->uio_iovcnt && count > 0) { 3255 /* 3256 * The end of the specified iovec's was reached but 3257 * the length requested could not be processed 3258 * (requested to digest more data than it provided). 3259 */ 3260 return (CRYPTO_DATA_LEN_RANGE); 3261 } 3262 break; 3263 3264 case CRYPTO_DATA_MBLK: 3265 /* 3266 * Jump to the first mblk_t containing data to be processed. 3267 */ 3268 for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp); 3269 off -= MBLKL(mp), mp = mp->b_cont) 3270 ; 3271 if (mp == NULL) { 3272 /* 3273 * The caller specified an offset that is larger than 3274 * the total size of the buffers it provided. 3275 */ 3276 return (CRYPTO_DATA_LEN_RANGE); 3277 } 3278 3279 /* 3280 * Now do the processing on the mblk chain. 3281 */ 3282 while (mp != NULL && count > 0) { 3283 cur_len = min(MBLKL(mp) - off, count); 3284 bcopy((char *)(mp->b_rptr + off), dest, cur_len); 3285 count -= cur_len; 3286 dest += cur_len; 3287 in->cd_offset += cur_len; 3288 in->cd_length -= cur_len; 3289 mp = mp->b_cont; 3290 off = 0; 3291 } 3292 3293 if (mp == NULL && count > 0) { 3294 /* 3295 * The end of the mblk was reached but the length 3296 * requested could not be processed, (requested to 3297 * digest more data than it provided). 3298 */ 3299 return (CRYPTO_DATA_LEN_RANGE); 3300 } 3301 break; 3302 3303 default: 3304 DBG(NULL, DWARN, 3305 "dca_resid_gather: unrecognised crypto data format"); 3306 rv = CRYPTO_ARGUMENTS_BAD; 3307 } 3308 return (rv); 3309 } 3310 3311 /* 3312 * Appends the data to the crypto_data_t struct increasing cd_length. 3313 * cd_offset is left unchanged. 3314 * Data is reverse-copied if the flag is TRUE. 3315 */ 3316 int 3317 dca_scatter(const char *src, crypto_data_t *out, int count, int reverse) 3318 { 3319 int rv = CRYPTO_SUCCESS; 3320 off_t offset = out->cd_offset + out->cd_length; 3321 uint_t vec_idx; 3322 uio_t *uiop; 3323 size_t cur_len; 3324 mblk_t *mp; 3325 3326 switch (out->cd_format) { 3327 case CRYPTO_DATA_RAW: 3328 if (out->cd_raw.iov_len - offset < count) { 3329 /* Trying to write out more than space available. */ 3330 return (CRYPTO_DATA_LEN_RANGE); 3331 } 3332 if (reverse) 3333 dca_reverse((void*) src, out->cd_raw.iov_base + offset, 3334 count, count); 3335 else 3336 bcopy(src, out->cd_raw.iov_base + offset, count); 3337 out->cd_length += count; 3338 break; 3339 3340 case CRYPTO_DATA_UIO: 3341 /* 3342 * Jump to the first iovec that can be written to. 3343 */ 3344 uiop = out->cd_uio; 3345 for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && 3346 offset >= uiop->uio_iov[vec_idx].iov_len; 3347 offset -= uiop->uio_iov[vec_idx++].iov_len) 3348 ; 3349 if (vec_idx == uiop->uio_iovcnt) { 3350 /* 3351 * The caller specified an offset that is larger than 3352 * the total size of the buffers it provided. 3353 */ 3354 return (CRYPTO_DATA_LEN_RANGE); 3355 } 3356 3357 /* 3358 * Now process the iovecs. 3359 */ 3360 while (vec_idx < uiop->uio_iovcnt && count > 0) { 3361 cur_len = min(uiop->uio_iov[vec_idx].iov_len - 3362 offset, count); 3363 count -= cur_len; 3364 if (reverse) { 3365 dca_reverse((void*) (src+count), 3366 uiop->uio_iov[vec_idx].iov_base + 3367 offset, cur_len, cur_len); 3368 } else { 3369 bcopy(src, uiop->uio_iov[vec_idx].iov_base + 3370 offset, cur_len); 3371 src += cur_len; 3372 } 3373 out->cd_length += cur_len; 3374 vec_idx++; 3375 offset = 0; 3376 } 3377 3378 if (vec_idx == uiop->uio_iovcnt && count > 0) { 3379 /* 3380 * The end of the specified iovec's was reached but 3381 * the length requested could not be processed 3382 * (requested to write more data than space provided). 3383 */ 3384 return (CRYPTO_DATA_LEN_RANGE); 3385 } 3386 break; 3387 3388 case CRYPTO_DATA_MBLK: 3389 /* 3390 * Jump to the first mblk_t that can be written to. 3391 */ 3392 for (mp = out->cd_mp; mp != NULL && offset >= MBLKL(mp); 3393 offset -= MBLKL(mp), mp = mp->b_cont) 3394 ; 3395 if (mp == NULL) { 3396 /* 3397 * The caller specified an offset that is larger than 3398 * the total size of the buffers it provided. 3399 */ 3400 return (CRYPTO_DATA_LEN_RANGE); 3401 } 3402 3403 /* 3404 * Now do the processing on the mblk chain. 3405 */ 3406 while (mp != NULL && count > 0) { 3407 cur_len = min(MBLKL(mp) - offset, count); 3408 count -= cur_len; 3409 if (reverse) { 3410 dca_reverse((void*) (src+count), 3411 (char *)(mp->b_rptr + offset), cur_len, 3412 cur_len); 3413 } else { 3414 bcopy(src, (char *)(mp->b_rptr + offset), 3415 cur_len); 3416 src += cur_len; 3417 } 3418 out->cd_length += cur_len; 3419 mp = mp->b_cont; 3420 offset = 0; 3421 } 3422 3423 if (mp == NULL && count > 0) { 3424 /* 3425 * The end of the mblk was reached but the length 3426 * requested could not be processed, (requested to 3427 * digest more data than it provided). 3428 */ 3429 return (CRYPTO_DATA_LEN_RANGE); 3430 } 3431 break; 3432 3433 default: 3434 DBG(NULL, DWARN, "unrecognised crypto data format"); 3435 rv = CRYPTO_ARGUMENTS_BAD; 3436 } 3437 return (rv); 3438 } 3439 3440 /* 3441 * Compare two byte arrays in reverse order. 3442 * Return 0 if they are identical, 1 otherwise. 3443 */ 3444 int 3445 dca_bcmp_reverse(const void *s1, const void *s2, size_t n) 3446 { 3447 int i; 3448 caddr_t src, dst; 3449 3450 if (!n) 3451 return (0); 3452 3453 src = ((caddr_t)s1) + n - 1; 3454 dst = (caddr_t)s2; 3455 for (i = 0; i < n; i++) { 3456 if (*src != *dst) 3457 return (1); 3458 src--; 3459 dst++; 3460 } 3461 3462 return (0); 3463 } 3464 3465 3466 /* 3467 * This calculates the size of a bignum in bits, specifically not counting 3468 * leading zero bits. This size calculation must be done *before* any 3469 * endian reversal takes place (i.e. the numbers are in absolute big-endian 3470 * order.) 3471 */ 3472 int 3473 dca_bitlen(unsigned char *bignum, int bytelen) 3474 { 3475 unsigned char msbyte; 3476 int i, j; 3477 3478 for (i = 0; i < bytelen - 1; i++) { 3479 if (bignum[i] != 0) { 3480 break; 3481 } 3482 } 3483 msbyte = bignum[i]; 3484 for (j = 8; j > 1; j--) { 3485 if (msbyte & 0x80) { 3486 break; 3487 } 3488 msbyte <<= 1; 3489 } 3490 return ((8 * (bytelen - i - 1)) + j); 3491 } 3492 3493 /* 3494 * This compares to bignums (in big-endian order). It ignores leading 3495 * null bytes. The result semantics follow bcmp, mempcmp, strcmp, etc. 3496 */ 3497 int 3498 dca_numcmp(caddr_t n1, int n1len, caddr_t n2, int n2len) 3499 { 3500 while ((n1len > 1) && (*n1 == 0)) { 3501 n1len--; 3502 n1++; 3503 } 3504 while ((n2len > 1) && (*n2 == 0)) { 3505 n2len--; 3506 n2++; 3507 } 3508 if (n1len != n2len) { 3509 return (n1len - n2len); 3510 } 3511 while ((n1len > 1) && (*n1 == *n2)) { 3512 n1++; 3513 n2++; 3514 n1len--; 3515 } 3516 return ((int)(*(uchar_t *)n1) - (int)(*(uchar_t *)n2)); 3517 } 3518 3519 /* 3520 * Return array of key attributes. 3521 */ 3522 crypto_object_attribute_t * 3523 dca_get_key_attr(crypto_key_t *key) 3524 { 3525 if ((key->ck_format != CRYPTO_KEY_ATTR_LIST) || 3526 (key->ck_count == 0)) { 3527 return (NULL); 3528 } 3529 3530 return (key->ck_attrs); 3531 } 3532 3533 /* 3534 * If attribute type exists valp points to it's 32-bit value. 3535 */ 3536 int 3537 dca_attr_lookup_uint32(crypto_object_attribute_t *attrp, uint_t atnum, 3538 uint64_t atype, uint32_t *valp) 3539 { 3540 crypto_object_attribute_t *bap; 3541 3542 bap = dca_find_attribute(attrp, atnum, atype); 3543 if (bap == NULL) { 3544 return (CRYPTO_ATTRIBUTE_TYPE_INVALID); 3545 } 3546 3547 *valp = *bap->oa_value; 3548 3549 return (CRYPTO_SUCCESS); 3550 } 3551 3552 /* 3553 * If attribute type exists data contains the start address of the value, 3554 * and numelems contains it's length. 3555 */ 3556 int 3557 dca_attr_lookup_uint8_array(crypto_object_attribute_t *attrp, uint_t atnum, 3558 uint64_t atype, void **data, unsigned int *numelems) 3559 { 3560 crypto_object_attribute_t *bap; 3561 3562 bap = dca_find_attribute(attrp, atnum, atype); 3563 if (bap == NULL) { 3564 return (CRYPTO_ATTRIBUTE_TYPE_INVALID); 3565 } 3566 3567 *data = bap->oa_value; 3568 *numelems = bap->oa_value_len; 3569 3570 return (CRYPTO_SUCCESS); 3571 } 3572 3573 /* 3574 * Finds entry of specified name. If it is not found dca_find_attribute returns 3575 * NULL. 3576 */ 3577 crypto_object_attribute_t * 3578 dca_find_attribute(crypto_object_attribute_t *attrp, uint_t atnum, 3579 uint64_t atype) 3580 { 3581 while (atnum) { 3582 if (attrp->oa_type == atype) 3583 return (attrp); 3584 atnum--; 3585 attrp++; 3586 } 3587 return (NULL); 3588 } 3589 3590 /* 3591 * Return the address of the first data buffer. If the data format is 3592 * unrecognised return NULL. 3593 */ 3594 caddr_t 3595 dca_bufdaddr(crypto_data_t *data) 3596 { 3597 switch (data->cd_format) { 3598 case CRYPTO_DATA_RAW: 3599 return (data->cd_raw.iov_base + data->cd_offset); 3600 case CRYPTO_DATA_UIO: 3601 return (data->cd_uio->uio_iov[0].iov_base + data->cd_offset); 3602 case CRYPTO_DATA_MBLK: 3603 return ((char *)data->cd_mp->b_rptr + data->cd_offset); 3604 default: 3605 DBG(NULL, DWARN, 3606 "dca_bufdaddr: unrecognised crypto data format"); 3607 return (NULL); 3608 } 3609 } 3610 3611 static caddr_t 3612 dca_bufdaddr_out(crypto_data_t *data) 3613 { 3614 size_t offset = data->cd_offset + data->cd_length; 3615 3616 switch (data->cd_format) { 3617 case CRYPTO_DATA_RAW: 3618 return (data->cd_raw.iov_base + offset); 3619 case CRYPTO_DATA_UIO: 3620 return (data->cd_uio->uio_iov[0].iov_base + offset); 3621 case CRYPTO_DATA_MBLK: 3622 return ((char *)data->cd_mp->b_rptr + offset); 3623 default: 3624 DBG(NULL, DWARN, 3625 "dca_bufdaddr_out: unrecognised crypto data format"); 3626 return (NULL); 3627 } 3628 } 3629 3630 /* 3631 * Control entry points. 3632 */ 3633 3634 /* ARGSUSED */ 3635 static void 3636 dca_provider_status(crypto_provider_handle_t provider, uint_t *status) 3637 { 3638 *status = CRYPTO_PROVIDER_READY; 3639 } 3640 3641 /* 3642 * Cipher (encrypt/decrypt) entry points. 3643 */ 3644 3645 /* ARGSUSED */ 3646 static int 3647 dca_encrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, 3648 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, 3649 crypto_req_handle_t req) 3650 { 3651 int error = CRYPTO_FAILED; 3652 dca_t *softc; 3653 3654 softc = DCA_SOFTC_FROM_CTX(ctx); 3655 DBG(softc, DENTRY, "dca_encrypt_init: started"); 3656 3657 /* check mechanism */ 3658 switch (mechanism->cm_type) { 3659 case DES_CBC_MECH_INFO_TYPE: 3660 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP, 3661 DR_ENCRYPT); 3662 break; 3663 case DES3_CBC_MECH_INFO_TYPE: 3664 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP, 3665 DR_ENCRYPT | DR_TRIPLE); 3666 break; 3667 case RSA_PKCS_MECH_INFO_TYPE: 3668 case RSA_X_509_MECH_INFO_TYPE: 3669 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); 3670 break; 3671 default: 3672 cmn_err(CE_WARN, "dca_encrypt_init: unexpected mech type " 3673 "0x%llx\n", (unsigned long long)mechanism->cm_type); 3674 error = CRYPTO_MECHANISM_INVALID; 3675 } 3676 3677 DBG(softc, DENTRY, "dca_encrypt_init: done, err = 0x%x", error); 3678 3679 if (error == CRYPTO_SUCCESS) 3680 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, 3681 &softc->dca_ctx_list_lock); 3682 3683 return (error); 3684 } 3685 3686 /* ARGSUSED */ 3687 static int 3688 dca_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext, 3689 crypto_data_t *ciphertext, crypto_req_handle_t req) 3690 { 3691 int error = CRYPTO_FAILED; 3692 dca_t *softc; 3693 3694 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 3695 return (CRYPTO_OPERATION_NOT_INITIALIZED); 3696 3697 softc = DCA_SOFTC_FROM_CTX(ctx); 3698 DBG(softc, DENTRY, "dca_encrypt: started"); 3699 3700 /* handle inplace ops */ 3701 if (!ciphertext) { 3702 dca_request_t *reqp = ctx->cc_provider_private; 3703 reqp->dr_flags |= DR_INPLACE; 3704 ciphertext = plaintext; 3705 } 3706 3707 /* check mechanism */ 3708 switch (DCA_MECH_FROM_CTX(ctx)) { 3709 case DES_CBC_MECH_INFO_TYPE: 3710 error = dca_3des(ctx, plaintext, ciphertext, req, DR_ENCRYPT); 3711 break; 3712 case DES3_CBC_MECH_INFO_TYPE: 3713 error = dca_3des(ctx, plaintext, ciphertext, req, 3714 DR_ENCRYPT | DR_TRIPLE); 3715 break; 3716 case RSA_PKCS_MECH_INFO_TYPE: 3717 case RSA_X_509_MECH_INFO_TYPE: 3718 error = dca_rsastart(ctx, plaintext, ciphertext, req, 3719 DCA_RSA_ENC); 3720 break; 3721 default: 3722 /* Should never reach here */ 3723 cmn_err(CE_WARN, "dca_encrypt: unexpected mech type " 3724 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 3725 error = CRYPTO_MECHANISM_INVALID; 3726 } 3727 3728 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) && 3729 (error != CRYPTO_BUFFER_TOO_SMALL)) { 3730 ciphertext->cd_length = 0; 3731 } 3732 3733 DBG(softc, DENTRY, "dca_encrypt: done, err = 0x%x", error); 3734 3735 return (error); 3736 } 3737 3738 /* ARGSUSED */ 3739 static int 3740 dca_encrypt_update(crypto_ctx_t *ctx, crypto_data_t *plaintext, 3741 crypto_data_t *ciphertext, crypto_req_handle_t req) 3742 { 3743 int error = CRYPTO_FAILED; 3744 dca_t *softc; 3745 3746 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 3747 return (CRYPTO_OPERATION_NOT_INITIALIZED); 3748 3749 softc = DCA_SOFTC_FROM_CTX(ctx); 3750 DBG(softc, DENTRY, "dca_encrypt_update: started"); 3751 3752 /* handle inplace ops */ 3753 if (!ciphertext) { 3754 dca_request_t *reqp = ctx->cc_provider_private; 3755 reqp->dr_flags |= DR_INPLACE; 3756 ciphertext = plaintext; 3757 } 3758 3759 /* check mechanism */ 3760 switch (DCA_MECH_FROM_CTX(ctx)) { 3761 case DES_CBC_MECH_INFO_TYPE: 3762 error = dca_3desupdate(ctx, plaintext, ciphertext, req, 3763 DR_ENCRYPT); 3764 break; 3765 case DES3_CBC_MECH_INFO_TYPE: 3766 error = dca_3desupdate(ctx, plaintext, ciphertext, req, 3767 DR_ENCRYPT | DR_TRIPLE); 3768 break; 3769 default: 3770 /* Should never reach here */ 3771 cmn_err(CE_WARN, "dca_encrypt_update: unexpected mech type " 3772 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 3773 error = CRYPTO_MECHANISM_INVALID; 3774 } 3775 3776 DBG(softc, DENTRY, "dca_encrypt_update: done, err = 0x%x", error); 3777 3778 return (error); 3779 } 3780 3781 /* ARGSUSED */ 3782 static int 3783 dca_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *ciphertext, 3784 crypto_req_handle_t req) 3785 { 3786 int error = CRYPTO_FAILED; 3787 dca_t *softc; 3788 3789 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 3790 return (CRYPTO_OPERATION_NOT_INITIALIZED); 3791 3792 softc = DCA_SOFTC_FROM_CTX(ctx); 3793 DBG(softc, DENTRY, "dca_encrypt_final: started"); 3794 3795 /* check mechanism */ 3796 switch (DCA_MECH_FROM_CTX(ctx)) { 3797 case DES_CBC_MECH_INFO_TYPE: 3798 error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT); 3799 break; 3800 case DES3_CBC_MECH_INFO_TYPE: 3801 error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT | DR_TRIPLE); 3802 break; 3803 default: 3804 /* Should never reach here */ 3805 cmn_err(CE_WARN, "dca_encrypt_final: unexpected mech type " 3806 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 3807 error = CRYPTO_MECHANISM_INVALID; 3808 } 3809 3810 DBG(softc, DENTRY, "dca_encrypt_final: done, err = 0x%x", error); 3811 3812 return (error); 3813 } 3814 3815 /* ARGSUSED */ 3816 static int 3817 dca_encrypt_atomic(crypto_provider_handle_t provider, 3818 crypto_session_id_t session_id, crypto_mechanism_t *mechanism, 3819 crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext, 3820 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) 3821 { 3822 int error = CRYPTO_FAILED; 3823 dca_t *softc = (dca_t *)provider; 3824 3825 DBG(softc, DENTRY, "dca_encrypt_atomic: started"); 3826 3827 if (ctx_template != NULL) 3828 return (CRYPTO_ARGUMENTS_BAD); 3829 3830 /* handle inplace ops */ 3831 if (!ciphertext) { 3832 ciphertext = plaintext; 3833 } 3834 3835 /* check mechanism */ 3836 switch (mechanism->cm_type) { 3837 case DES_CBC_MECH_INFO_TYPE: 3838 error = dca_3desatomic(provider, session_id, mechanism, key, 3839 plaintext, ciphertext, KM_SLEEP, req, 3840 DR_ENCRYPT | DR_ATOMIC); 3841 break; 3842 case DES3_CBC_MECH_INFO_TYPE: 3843 error = dca_3desatomic(provider, session_id, mechanism, key, 3844 plaintext, ciphertext, KM_SLEEP, req, 3845 DR_ENCRYPT | DR_TRIPLE | DR_ATOMIC); 3846 break; 3847 case RSA_PKCS_MECH_INFO_TYPE: 3848 case RSA_X_509_MECH_INFO_TYPE: 3849 error = dca_rsaatomic(provider, session_id, mechanism, key, 3850 plaintext, ciphertext, KM_SLEEP, req, DCA_RSA_ENC); 3851 break; 3852 default: 3853 cmn_err(CE_WARN, "dca_encrypt_atomic: unexpected mech type " 3854 "0x%llx\n", (unsigned long long)mechanism->cm_type); 3855 error = CRYPTO_MECHANISM_INVALID; 3856 } 3857 3858 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) { 3859 ciphertext->cd_length = 0; 3860 } 3861 3862 DBG(softc, DENTRY, "dca_encrypt_atomic: done, err = 0x%x", error); 3863 3864 return (error); 3865 } 3866 3867 /* ARGSUSED */ 3868 static int 3869 dca_decrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, 3870 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, 3871 crypto_req_handle_t req) 3872 { 3873 int error = CRYPTO_FAILED; 3874 dca_t *softc; 3875 3876 softc = DCA_SOFTC_FROM_CTX(ctx); 3877 DBG(softc, DENTRY, "dca_decrypt_init: started"); 3878 3879 /* check mechanism */ 3880 switch (mechanism->cm_type) { 3881 case DES_CBC_MECH_INFO_TYPE: 3882 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP, 3883 DR_DECRYPT); 3884 break; 3885 case DES3_CBC_MECH_INFO_TYPE: 3886 error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP, 3887 DR_DECRYPT | DR_TRIPLE); 3888 break; 3889 case RSA_PKCS_MECH_INFO_TYPE: 3890 case RSA_X_509_MECH_INFO_TYPE: 3891 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); 3892 break; 3893 default: 3894 cmn_err(CE_WARN, "dca_decrypt_init: unexpected mech type " 3895 "0x%llx\n", (unsigned long long)mechanism->cm_type); 3896 error = CRYPTO_MECHANISM_INVALID; 3897 } 3898 3899 DBG(softc, DENTRY, "dca_decrypt_init: done, err = 0x%x", error); 3900 3901 if (error == CRYPTO_SUCCESS) 3902 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, 3903 &softc->dca_ctx_list_lock); 3904 3905 return (error); 3906 } 3907 3908 /* ARGSUSED */ 3909 static int 3910 dca_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext, 3911 crypto_data_t *plaintext, crypto_req_handle_t req) 3912 { 3913 int error = CRYPTO_FAILED; 3914 dca_t *softc; 3915 3916 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 3917 return (CRYPTO_OPERATION_NOT_INITIALIZED); 3918 3919 softc = DCA_SOFTC_FROM_CTX(ctx); 3920 DBG(softc, DENTRY, "dca_decrypt: started"); 3921 3922 /* handle inplace ops */ 3923 if (!plaintext) { 3924 dca_request_t *reqp = ctx->cc_provider_private; 3925 reqp->dr_flags |= DR_INPLACE; 3926 plaintext = ciphertext; 3927 } 3928 3929 /* check mechanism */ 3930 switch (DCA_MECH_FROM_CTX(ctx)) { 3931 case DES_CBC_MECH_INFO_TYPE: 3932 error = dca_3des(ctx, ciphertext, plaintext, req, DR_DECRYPT); 3933 break; 3934 case DES3_CBC_MECH_INFO_TYPE: 3935 error = dca_3des(ctx, ciphertext, plaintext, req, 3936 DR_DECRYPT | DR_TRIPLE); 3937 break; 3938 case RSA_PKCS_MECH_INFO_TYPE: 3939 case RSA_X_509_MECH_INFO_TYPE: 3940 error = dca_rsastart(ctx, ciphertext, plaintext, req, 3941 DCA_RSA_DEC); 3942 break; 3943 default: 3944 /* Should never reach here */ 3945 cmn_err(CE_WARN, "dca_decrypt: unexpected mech type " 3946 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 3947 error = CRYPTO_MECHANISM_INVALID; 3948 } 3949 3950 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) && 3951 (error != CRYPTO_BUFFER_TOO_SMALL)) { 3952 if (plaintext) 3953 plaintext->cd_length = 0; 3954 } 3955 3956 DBG(softc, DENTRY, "dca_decrypt: done, err = 0x%x", error); 3957 3958 return (error); 3959 } 3960 3961 /* ARGSUSED */ 3962 static int 3963 dca_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext, 3964 crypto_data_t *plaintext, crypto_req_handle_t req) 3965 { 3966 int error = CRYPTO_FAILED; 3967 dca_t *softc; 3968 3969 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 3970 return (CRYPTO_OPERATION_NOT_INITIALIZED); 3971 3972 softc = DCA_SOFTC_FROM_CTX(ctx); 3973 DBG(softc, DENTRY, "dca_decrypt_update: started"); 3974 3975 /* handle inplace ops */ 3976 if (!plaintext) { 3977 dca_request_t *reqp = ctx->cc_provider_private; 3978 reqp->dr_flags |= DR_INPLACE; 3979 plaintext = ciphertext; 3980 } 3981 3982 /* check mechanism */ 3983 switch (DCA_MECH_FROM_CTX(ctx)) { 3984 case DES_CBC_MECH_INFO_TYPE: 3985 error = dca_3desupdate(ctx, ciphertext, plaintext, req, 3986 DR_DECRYPT); 3987 break; 3988 case DES3_CBC_MECH_INFO_TYPE: 3989 error = dca_3desupdate(ctx, ciphertext, plaintext, req, 3990 DR_DECRYPT | DR_TRIPLE); 3991 break; 3992 default: 3993 /* Should never reach here */ 3994 cmn_err(CE_WARN, "dca_decrypt_update: unexpected mech type " 3995 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 3996 error = CRYPTO_MECHANISM_INVALID; 3997 } 3998 3999 DBG(softc, DENTRY, "dca_decrypt_update: done, err = 0x%x", error); 4000 4001 return (error); 4002 } 4003 4004 /* ARGSUSED */ 4005 static int 4006 dca_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *plaintext, 4007 crypto_req_handle_t req) 4008 { 4009 int error = CRYPTO_FAILED; 4010 dca_t *softc; 4011 4012 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4013 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4014 4015 softc = DCA_SOFTC_FROM_CTX(ctx); 4016 DBG(softc, DENTRY, "dca_decrypt_final: started"); 4017 4018 /* check mechanism */ 4019 switch (DCA_MECH_FROM_CTX(ctx)) { 4020 case DES_CBC_MECH_INFO_TYPE: 4021 error = dca_3desfinal(ctx, plaintext, DR_DECRYPT); 4022 break; 4023 case DES3_CBC_MECH_INFO_TYPE: 4024 error = dca_3desfinal(ctx, plaintext, DR_DECRYPT | DR_TRIPLE); 4025 break; 4026 default: 4027 /* Should never reach here */ 4028 cmn_err(CE_WARN, "dca_decrypt_final: unexpected mech type " 4029 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4030 error = CRYPTO_MECHANISM_INVALID; 4031 } 4032 4033 DBG(softc, DENTRY, "dca_decrypt_final: done, err = 0x%x", error); 4034 4035 return (error); 4036 } 4037 4038 /* ARGSUSED */ 4039 static int 4040 dca_decrypt_atomic(crypto_provider_handle_t provider, 4041 crypto_session_id_t session_id, crypto_mechanism_t *mechanism, 4042 crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext, 4043 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) 4044 { 4045 int error = CRYPTO_FAILED; 4046 dca_t *softc = (dca_t *)provider; 4047 4048 DBG(softc, DENTRY, "dca_decrypt_atomic: started"); 4049 4050 if (ctx_template != NULL) 4051 return (CRYPTO_ARGUMENTS_BAD); 4052 4053 /* handle inplace ops */ 4054 if (!plaintext) { 4055 plaintext = ciphertext; 4056 } 4057 4058 /* check mechanism */ 4059 switch (mechanism->cm_type) { 4060 case DES_CBC_MECH_INFO_TYPE: 4061 error = dca_3desatomic(provider, session_id, mechanism, key, 4062 ciphertext, plaintext, KM_SLEEP, req, 4063 DR_DECRYPT | DR_ATOMIC); 4064 break; 4065 case DES3_CBC_MECH_INFO_TYPE: 4066 error = dca_3desatomic(provider, session_id, mechanism, key, 4067 ciphertext, plaintext, KM_SLEEP, req, 4068 DR_DECRYPT | DR_TRIPLE | DR_ATOMIC); 4069 break; 4070 case RSA_PKCS_MECH_INFO_TYPE: 4071 case RSA_X_509_MECH_INFO_TYPE: 4072 error = dca_rsaatomic(provider, session_id, mechanism, key, 4073 ciphertext, plaintext, KM_SLEEP, req, DCA_RSA_DEC); 4074 break; 4075 default: 4076 cmn_err(CE_WARN, "dca_decrypt_atomic: unexpected mech type " 4077 "0x%llx\n", (unsigned long long)mechanism->cm_type); 4078 error = CRYPTO_MECHANISM_INVALID; 4079 } 4080 4081 if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) { 4082 plaintext->cd_length = 0; 4083 } 4084 4085 DBG(softc, DENTRY, "dca_decrypt_atomic: done, err = 0x%x", error); 4086 4087 return (error); 4088 } 4089 4090 /* 4091 * Sign entry points. 4092 */ 4093 4094 /* ARGSUSED */ 4095 static int 4096 dca_sign_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, 4097 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, 4098 crypto_req_handle_t req) 4099 { 4100 int error = CRYPTO_FAILED; 4101 dca_t *softc; 4102 4103 softc = DCA_SOFTC_FROM_CTX(ctx); 4104 DBG(softc, DENTRY, "dca_sign_init: started\n"); 4105 4106 if (ctx_template != NULL) 4107 return (CRYPTO_ARGUMENTS_BAD); 4108 4109 /* check mechanism */ 4110 switch (mechanism->cm_type) { 4111 case RSA_PKCS_MECH_INFO_TYPE: 4112 case RSA_X_509_MECH_INFO_TYPE: 4113 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); 4114 break; 4115 case DSA_MECH_INFO_TYPE: 4116 error = dca_dsainit(ctx, mechanism, key, KM_SLEEP, 4117 DCA_DSA_SIGN); 4118 break; 4119 default: 4120 cmn_err(CE_WARN, "dca_sign_init: unexpected mech type " 4121 "0x%llx\n", (unsigned long long)mechanism->cm_type); 4122 error = CRYPTO_MECHANISM_INVALID; 4123 } 4124 4125 DBG(softc, DENTRY, "dca_sign_init: done, err = 0x%x", error); 4126 4127 if (error == CRYPTO_SUCCESS) 4128 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, 4129 &softc->dca_ctx_list_lock); 4130 4131 return (error); 4132 } 4133 4134 static int 4135 dca_sign(crypto_ctx_t *ctx, crypto_data_t *data, 4136 crypto_data_t *signature, crypto_req_handle_t req) 4137 { 4138 int error = CRYPTO_FAILED; 4139 dca_t *softc; 4140 4141 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4142 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4143 4144 softc = DCA_SOFTC_FROM_CTX(ctx); 4145 DBG(softc, DENTRY, "dca_sign: started\n"); 4146 4147 /* check mechanism */ 4148 switch (DCA_MECH_FROM_CTX(ctx)) { 4149 case RSA_PKCS_MECH_INFO_TYPE: 4150 case RSA_X_509_MECH_INFO_TYPE: 4151 error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGN); 4152 break; 4153 case DSA_MECH_INFO_TYPE: 4154 error = dca_dsa_sign(ctx, data, signature, req); 4155 break; 4156 default: 4157 cmn_err(CE_WARN, "dca_sign: unexpected mech type " 4158 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4159 error = CRYPTO_MECHANISM_INVALID; 4160 } 4161 4162 DBG(softc, DENTRY, "dca_sign: done, err = 0x%x", error); 4163 4164 return (error); 4165 } 4166 4167 /* ARGSUSED */ 4168 static int 4169 dca_sign_update(crypto_ctx_t *ctx, crypto_data_t *data, 4170 crypto_req_handle_t req) 4171 { 4172 int error = CRYPTO_MECHANISM_INVALID; 4173 dca_t *softc; 4174 4175 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4176 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4177 4178 softc = DCA_SOFTC_FROM_CTX(ctx); 4179 DBG(softc, DENTRY, "dca_sign_update: started\n"); 4180 4181 cmn_err(CE_WARN, "dca_sign_update: unexpected mech type " 4182 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4183 4184 DBG(softc, DENTRY, "dca_sign_update: done, err = 0x%x", error); 4185 4186 return (error); 4187 } 4188 4189 /* ARGSUSED */ 4190 static int 4191 dca_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature, 4192 crypto_req_handle_t req) 4193 { 4194 int error = CRYPTO_MECHANISM_INVALID; 4195 dca_t *softc; 4196 4197 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4198 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4199 4200 softc = DCA_SOFTC_FROM_CTX(ctx); 4201 DBG(softc, DENTRY, "dca_sign_final: started\n"); 4202 4203 cmn_err(CE_WARN, "dca_sign_final: unexpected mech type " 4204 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4205 4206 DBG(softc, DENTRY, "dca_sign_final: done, err = 0x%x", error); 4207 4208 return (error); 4209 } 4210 4211 static int 4212 dca_sign_atomic(crypto_provider_handle_t provider, 4213 crypto_session_id_t session_id, crypto_mechanism_t *mechanism, 4214 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature, 4215 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) 4216 { 4217 int error = CRYPTO_FAILED; 4218 dca_t *softc = (dca_t *)provider; 4219 4220 DBG(softc, DENTRY, "dca_sign_atomic: started\n"); 4221 4222 if (ctx_template != NULL) 4223 return (CRYPTO_ARGUMENTS_BAD); 4224 4225 /* check mechanism */ 4226 switch (mechanism->cm_type) { 4227 case RSA_PKCS_MECH_INFO_TYPE: 4228 case RSA_X_509_MECH_INFO_TYPE: 4229 error = dca_rsaatomic(provider, session_id, mechanism, key, 4230 data, signature, KM_SLEEP, req, DCA_RSA_SIGN); 4231 break; 4232 case DSA_MECH_INFO_TYPE: 4233 error = dca_dsaatomic(provider, session_id, mechanism, key, 4234 data, signature, KM_SLEEP, req, DCA_DSA_SIGN); 4235 break; 4236 default: 4237 cmn_err(CE_WARN, "dca_sign_atomic: unexpected mech type " 4238 "0x%llx\n", (unsigned long long)mechanism->cm_type); 4239 error = CRYPTO_MECHANISM_INVALID; 4240 } 4241 4242 DBG(softc, DENTRY, "dca_sign_atomic: done, err = 0x%x", error); 4243 4244 return (error); 4245 } 4246 4247 /* ARGSUSED */ 4248 static int 4249 dca_sign_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, 4250 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, 4251 crypto_req_handle_t req) 4252 { 4253 int error = CRYPTO_FAILED; 4254 dca_t *softc; 4255 4256 softc = DCA_SOFTC_FROM_CTX(ctx); 4257 DBG(softc, DENTRY, "dca_sign_recover_init: started\n"); 4258 4259 if (ctx_template != NULL) 4260 return (CRYPTO_ARGUMENTS_BAD); 4261 4262 /* check mechanism */ 4263 switch (mechanism->cm_type) { 4264 case RSA_PKCS_MECH_INFO_TYPE: 4265 case RSA_X_509_MECH_INFO_TYPE: 4266 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); 4267 break; 4268 default: 4269 cmn_err(CE_WARN, "dca_sign_recover_init: unexpected mech type " 4270 "0x%llx\n", (unsigned long long)mechanism->cm_type); 4271 error = CRYPTO_MECHANISM_INVALID; 4272 } 4273 4274 DBG(softc, DENTRY, "dca_sign_recover_init: done, err = 0x%x", error); 4275 4276 if (error == CRYPTO_SUCCESS) 4277 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, 4278 &softc->dca_ctx_list_lock); 4279 4280 return (error); 4281 } 4282 4283 static int 4284 dca_sign_recover(crypto_ctx_t *ctx, crypto_data_t *data, 4285 crypto_data_t *signature, crypto_req_handle_t req) 4286 { 4287 int error = CRYPTO_FAILED; 4288 dca_t *softc; 4289 4290 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4291 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4292 4293 softc = DCA_SOFTC_FROM_CTX(ctx); 4294 DBG(softc, DENTRY, "dca_sign_recover: started\n"); 4295 4296 /* check mechanism */ 4297 switch (DCA_MECH_FROM_CTX(ctx)) { 4298 case RSA_PKCS_MECH_INFO_TYPE: 4299 case RSA_X_509_MECH_INFO_TYPE: 4300 error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGNR); 4301 break; 4302 default: 4303 cmn_err(CE_WARN, "dca_sign_recover: unexpected mech type " 4304 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4305 error = CRYPTO_MECHANISM_INVALID; 4306 } 4307 4308 DBG(softc, DENTRY, "dca_sign_recover: done, err = 0x%x", error); 4309 4310 return (error); 4311 } 4312 4313 static int 4314 dca_sign_recover_atomic(crypto_provider_handle_t provider, 4315 crypto_session_id_t session_id, crypto_mechanism_t *mechanism, 4316 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature, 4317 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) 4318 { 4319 int error = CRYPTO_FAILED; 4320 dca_t *softc = (dca_t *)provider; 4321 4322 DBG(softc, DENTRY, "dca_sign_recover_atomic: started\n"); 4323 4324 if (ctx_template != NULL) 4325 return (CRYPTO_ARGUMENTS_BAD); 4326 4327 /* check mechanism */ 4328 switch (mechanism->cm_type) { 4329 case RSA_PKCS_MECH_INFO_TYPE: 4330 case RSA_X_509_MECH_INFO_TYPE: 4331 error = dca_rsaatomic(provider, session_id, mechanism, key, 4332 data, signature, KM_SLEEP, req, DCA_RSA_SIGNR); 4333 break; 4334 default: 4335 cmn_err(CE_WARN, "dca_sign_recover_atomic: unexpected mech type" 4336 " 0x%llx\n", (unsigned long long)mechanism->cm_type); 4337 error = CRYPTO_MECHANISM_INVALID; 4338 } 4339 4340 DBG(softc, DENTRY, "dca_sign_recover_atomic: done, err = 0x%x", error); 4341 4342 return (error); 4343 } 4344 4345 /* 4346 * Verify entry points. 4347 */ 4348 4349 /* ARGSUSED */ 4350 static int 4351 dca_verify_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, 4352 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, 4353 crypto_req_handle_t req) 4354 { 4355 int error = CRYPTO_FAILED; 4356 dca_t *softc; 4357 4358 softc = DCA_SOFTC_FROM_CTX(ctx); 4359 DBG(softc, DENTRY, "dca_verify_init: started\n"); 4360 4361 if (ctx_template != NULL) 4362 return (CRYPTO_ARGUMENTS_BAD); 4363 4364 /* check mechanism */ 4365 switch (mechanism->cm_type) { 4366 case RSA_PKCS_MECH_INFO_TYPE: 4367 case RSA_X_509_MECH_INFO_TYPE: 4368 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); 4369 break; 4370 case DSA_MECH_INFO_TYPE: 4371 error = dca_dsainit(ctx, mechanism, key, KM_SLEEP, 4372 DCA_DSA_VRFY); 4373 break; 4374 default: 4375 cmn_err(CE_WARN, "dca_verify_init: unexpected mech type " 4376 "0x%llx\n", (unsigned long long)mechanism->cm_type); 4377 error = CRYPTO_MECHANISM_INVALID; 4378 } 4379 4380 DBG(softc, DENTRY, "dca_verify_init: done, err = 0x%x", error); 4381 4382 if (error == CRYPTO_SUCCESS) 4383 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, 4384 &softc->dca_ctx_list_lock); 4385 4386 return (error); 4387 } 4388 4389 static int 4390 dca_verify(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature, 4391 crypto_req_handle_t req) 4392 { 4393 int error = CRYPTO_FAILED; 4394 dca_t *softc; 4395 4396 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4397 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4398 4399 softc = DCA_SOFTC_FROM_CTX(ctx); 4400 DBG(softc, DENTRY, "dca_verify: started\n"); 4401 4402 /* check mechanism */ 4403 switch (DCA_MECH_FROM_CTX(ctx)) { 4404 case RSA_PKCS_MECH_INFO_TYPE: 4405 case RSA_X_509_MECH_INFO_TYPE: 4406 error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFY); 4407 break; 4408 case DSA_MECH_INFO_TYPE: 4409 error = dca_dsa_verify(ctx, data, signature, req); 4410 break; 4411 default: 4412 cmn_err(CE_WARN, "dca_verify: unexpected mech type " 4413 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4414 error = CRYPTO_MECHANISM_INVALID; 4415 } 4416 4417 DBG(softc, DENTRY, "dca_verify: done, err = 0x%x", error); 4418 4419 return (error); 4420 } 4421 4422 /* ARGSUSED */ 4423 static int 4424 dca_verify_update(crypto_ctx_t *ctx, crypto_data_t *data, 4425 crypto_req_handle_t req) 4426 { 4427 int error = CRYPTO_MECHANISM_INVALID; 4428 dca_t *softc; 4429 4430 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4431 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4432 4433 softc = DCA_SOFTC_FROM_CTX(ctx); 4434 DBG(softc, DENTRY, "dca_verify_update: started\n"); 4435 4436 cmn_err(CE_WARN, "dca_verify_update: unexpected mech type " 4437 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4438 4439 DBG(softc, DENTRY, "dca_verify_update: done, err = 0x%x", error); 4440 4441 return (error); 4442 } 4443 4444 /* ARGSUSED */ 4445 static int 4446 dca_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature, 4447 crypto_req_handle_t req) 4448 { 4449 int error = CRYPTO_MECHANISM_INVALID; 4450 dca_t *softc; 4451 4452 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4453 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4454 4455 softc = DCA_SOFTC_FROM_CTX(ctx); 4456 DBG(softc, DENTRY, "dca_verify_final: started\n"); 4457 4458 cmn_err(CE_WARN, "dca_verify_final: unexpected mech type " 4459 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4460 4461 DBG(softc, DENTRY, "dca_verify_final: done, err = 0x%x", error); 4462 4463 return (error); 4464 } 4465 4466 static int 4467 dca_verify_atomic(crypto_provider_handle_t provider, 4468 crypto_session_id_t session_id, crypto_mechanism_t *mechanism, 4469 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature, 4470 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) 4471 { 4472 int error = CRYPTO_FAILED; 4473 dca_t *softc = (dca_t *)provider; 4474 4475 DBG(softc, DENTRY, "dca_verify_atomic: started\n"); 4476 4477 if (ctx_template != NULL) 4478 return (CRYPTO_ARGUMENTS_BAD); 4479 4480 /* check mechanism */ 4481 switch (mechanism->cm_type) { 4482 case RSA_PKCS_MECH_INFO_TYPE: 4483 case RSA_X_509_MECH_INFO_TYPE: 4484 error = dca_rsaatomic(provider, session_id, mechanism, key, 4485 signature, data, KM_SLEEP, req, DCA_RSA_VRFY); 4486 break; 4487 case DSA_MECH_INFO_TYPE: 4488 error = dca_dsaatomic(provider, session_id, mechanism, key, 4489 data, signature, KM_SLEEP, req, DCA_DSA_VRFY); 4490 break; 4491 default: 4492 cmn_err(CE_WARN, "dca_verify_atomic: unexpected mech type " 4493 "0x%llx\n", (unsigned long long)mechanism->cm_type); 4494 error = CRYPTO_MECHANISM_INVALID; 4495 } 4496 4497 DBG(softc, DENTRY, "dca_verify_atomic: done, err = 0x%x", error); 4498 4499 return (error); 4500 } 4501 4502 /* ARGSUSED */ 4503 static int 4504 dca_verify_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, 4505 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, 4506 crypto_req_handle_t req) 4507 { 4508 int error = CRYPTO_MECHANISM_INVALID; 4509 dca_t *softc; 4510 4511 softc = DCA_SOFTC_FROM_CTX(ctx); 4512 DBG(softc, DENTRY, "dca_verify_recover_init: started\n"); 4513 4514 if (ctx_template != NULL) 4515 return (CRYPTO_ARGUMENTS_BAD); 4516 4517 /* check mechanism */ 4518 switch (mechanism->cm_type) { 4519 case RSA_PKCS_MECH_INFO_TYPE: 4520 case RSA_X_509_MECH_INFO_TYPE: 4521 error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); 4522 break; 4523 default: 4524 cmn_err(CE_WARN, "dca_verify_recover_init: unexpected mech type" 4525 " 0x%llx\n", (unsigned long long)mechanism->cm_type); 4526 } 4527 4528 DBG(softc, DENTRY, "dca_verify_recover_init: done, err = 0x%x", error); 4529 4530 if (error == CRYPTO_SUCCESS) 4531 dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, 4532 &softc->dca_ctx_list_lock); 4533 4534 return (error); 4535 } 4536 4537 static int 4538 dca_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature, 4539 crypto_data_t *data, crypto_req_handle_t req) 4540 { 4541 int error = CRYPTO_MECHANISM_INVALID; 4542 dca_t *softc; 4543 4544 if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) 4545 return (CRYPTO_OPERATION_NOT_INITIALIZED); 4546 4547 softc = DCA_SOFTC_FROM_CTX(ctx); 4548 DBG(softc, DENTRY, "dca_verify_recover: started\n"); 4549 4550 /* check mechanism */ 4551 switch (DCA_MECH_FROM_CTX(ctx)) { 4552 case RSA_PKCS_MECH_INFO_TYPE: 4553 case RSA_X_509_MECH_INFO_TYPE: 4554 error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFYR); 4555 break; 4556 default: 4557 cmn_err(CE_WARN, "dca_verify_recover: unexpected mech type " 4558 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4559 } 4560 4561 DBG(softc, DENTRY, "dca_verify_recover: done, err = 0x%x", error); 4562 4563 return (error); 4564 } 4565 4566 static int 4567 dca_verify_recover_atomic(crypto_provider_handle_t provider, 4568 crypto_session_id_t session_id, crypto_mechanism_t *mechanism, 4569 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature, 4570 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) 4571 { 4572 int error = CRYPTO_MECHANISM_INVALID; 4573 dca_t *softc = (dca_t *)provider; 4574 4575 DBG(softc, DENTRY, "dca_verify_recover_atomic: started\n"); 4576 4577 if (ctx_template != NULL) 4578 return (CRYPTO_ARGUMENTS_BAD); 4579 4580 /* check mechanism */ 4581 switch (mechanism->cm_type) { 4582 case RSA_PKCS_MECH_INFO_TYPE: 4583 case RSA_X_509_MECH_INFO_TYPE: 4584 error = dca_rsaatomic(provider, session_id, mechanism, key, 4585 signature, data, KM_SLEEP, req, DCA_RSA_VRFYR); 4586 break; 4587 default: 4588 cmn_err(CE_WARN, "dca_verify_recover_atomic: unexpected mech " 4589 "type 0x%llx\n", (unsigned long long)mechanism->cm_type); 4590 error = CRYPTO_MECHANISM_INVALID; 4591 } 4592 4593 DBG(softc, DENTRY, 4594 "dca_verify_recover_atomic: done, err = 0x%x", error); 4595 4596 return (error); 4597 } 4598 4599 /* 4600 * Random number entry points. 4601 */ 4602 4603 /* ARGSUSED */ 4604 static int 4605 dca_generate_random(crypto_provider_handle_t provider, 4606 crypto_session_id_t session_id, 4607 uchar_t *buf, size_t len, crypto_req_handle_t req) 4608 { 4609 int error = CRYPTO_FAILED; 4610 dca_t *softc = (dca_t *)provider; 4611 4612 DBG(softc, DENTRY, "dca_generate_random: started"); 4613 4614 error = dca_rng(softc, buf, len, req); 4615 4616 DBG(softc, DENTRY, "dca_generate_random: done, err = 0x%x", error); 4617 4618 return (error); 4619 } 4620 4621 /* 4622 * Context management entry points. 4623 */ 4624 4625 int 4626 dca_free_context(crypto_ctx_t *ctx) 4627 { 4628 int error = CRYPTO_SUCCESS; 4629 dca_t *softc; 4630 4631 softc = DCA_SOFTC_FROM_CTX(ctx); 4632 DBG(softc, DENTRY, "dca_free_context: entered"); 4633 4634 if (ctx->cc_provider_private == NULL) 4635 return (error); 4636 4637 dca_rmlist2(ctx->cc_provider_private, &softc->dca_ctx_list_lock); 4638 4639 error = dca_free_context_low(ctx); 4640 4641 DBG(softc, DENTRY, "dca_free_context: done, err = 0x%x", error); 4642 4643 return (error); 4644 } 4645 4646 static int 4647 dca_free_context_low(crypto_ctx_t *ctx) 4648 { 4649 int error = CRYPTO_SUCCESS; 4650 4651 /* check mechanism */ 4652 switch (DCA_MECH_FROM_CTX(ctx)) { 4653 case DES_CBC_MECH_INFO_TYPE: 4654 case DES3_CBC_MECH_INFO_TYPE: 4655 dca_3desctxfree(ctx); 4656 break; 4657 case RSA_PKCS_MECH_INFO_TYPE: 4658 case RSA_X_509_MECH_INFO_TYPE: 4659 dca_rsactxfree(ctx); 4660 break; 4661 case DSA_MECH_INFO_TYPE: 4662 dca_dsactxfree(ctx); 4663 break; 4664 default: 4665 /* Should never reach here */ 4666 cmn_err(CE_WARN, "dca_free_context_low: unexpected mech type " 4667 "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); 4668 error = CRYPTO_MECHANISM_INVALID; 4669 } 4670 4671 return (error); 4672 } 4673 4674 4675 /* Free any unfreed private context. It is called in detach. */ 4676 static void 4677 dca_free_context_list(dca_t *dca) 4678 { 4679 dca_listnode_t *node; 4680 crypto_ctx_t ctx; 4681 4682 (void) memset(&ctx, 0, sizeof (ctx)); 4683 ctx.cc_provider = dca; 4684 4685 while ((node = dca_delist2(&dca->dca_ctx_list, 4686 &dca->dca_ctx_list_lock)) != NULL) { 4687 ctx.cc_provider_private = node; 4688 (void) dca_free_context_low(&ctx); 4689 } 4690 } 4691 4692 static int 4693 ext_info_sym(crypto_provider_handle_t prov, 4694 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq) 4695 { 4696 return (ext_info_base(prov, ext_info, cfreq, IDENT_SYM)); 4697 } 4698 4699 static int 4700 ext_info_asym(crypto_provider_handle_t prov, 4701 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq) 4702 { 4703 int rv; 4704 4705 rv = ext_info_base(prov, ext_info, cfreq, IDENT_ASYM); 4706 /* The asymmetric cipher slot supports random */ 4707 ext_info->ei_flags |= CRYPTO_EXTF_RNG; 4708 4709 return (rv); 4710 } 4711 4712 /* ARGSUSED */ 4713 static int 4714 ext_info_base(crypto_provider_handle_t prov, 4715 crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id) 4716 { 4717 dca_t *dca = (dca_t *)prov; 4718 int len; 4719 4720 /* Label */ 4721 (void) sprintf((char *)ext_info->ei_label, "%s/%d %s", 4722 ddi_driver_name(dca->dca_dip), ddi_get_instance(dca->dca_dip), id); 4723 len = strlen((char *)ext_info->ei_label); 4724 (void) memset(ext_info->ei_label + len, ' ', 4725 CRYPTO_EXT_SIZE_LABEL - len); 4726 4727 /* Manufacturer ID */ 4728 (void) sprintf((char *)ext_info->ei_manufacturerID, "%s", 4729 DCA_MANUFACTURER_ID); 4730 len = strlen((char *)ext_info->ei_manufacturerID); 4731 (void) memset(ext_info->ei_manufacturerID + len, ' ', 4732 CRYPTO_EXT_SIZE_MANUF - len); 4733 4734 /* Model */ 4735 (void) sprintf((char *)ext_info->ei_model, dca->dca_model); 4736 4737 DBG(dca, DWARN, "kCF MODEL: %s", (char *)ext_info->ei_model); 4738 4739 len = strlen((char *)ext_info->ei_model); 4740 (void) memset(ext_info->ei_model + len, ' ', 4741 CRYPTO_EXT_SIZE_MODEL - len); 4742 4743 /* Serial Number. Blank for Deimos */ 4744 (void) memset(ext_info->ei_serial_number, ' ', CRYPTO_EXT_SIZE_SERIAL); 4745 4746 ext_info->ei_flags = CRYPTO_EXTF_WRITE_PROTECTED; 4747 4748 ext_info->ei_max_session_count = CRYPTO_UNAVAILABLE_INFO; 4749 ext_info->ei_max_pin_len = CRYPTO_UNAVAILABLE_INFO; 4750 ext_info->ei_min_pin_len = CRYPTO_UNAVAILABLE_INFO; 4751 ext_info->ei_total_public_memory = CRYPTO_UNAVAILABLE_INFO; 4752 ext_info->ei_free_public_memory = CRYPTO_UNAVAILABLE_INFO; 4753 ext_info->ei_total_private_memory = CRYPTO_UNAVAILABLE_INFO; 4754 ext_info->ei_free_private_memory = CRYPTO_UNAVAILABLE_INFO; 4755 ext_info->ei_hardware_version.cv_major = 0; 4756 ext_info->ei_hardware_version.cv_minor = 0; 4757 ext_info->ei_firmware_version.cv_major = 0; 4758 ext_info->ei_firmware_version.cv_minor = 0; 4759 4760 /* Time. No need to be supplied for token without a clock */ 4761 ext_info->ei_time[0] = '\000'; 4762 4763 return (CRYPTO_SUCCESS); 4764 } 4765 4766 static void 4767 dca_fma_init(dca_t *dca) 4768 { 4769 ddi_iblock_cookie_t fm_ibc; 4770 int fm_capabilities = DDI_FM_EREPORT_CAPABLE | 4771 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE | 4772 DDI_FM_ERRCB_CAPABLE; 4773 4774 /* Read FMA capabilities from dca.conf file (if present) */ 4775 dca->fm_capabilities = ddi_getprop(DDI_DEV_T_ANY, dca->dca_dip, 4776 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", 4777 fm_capabilities); 4778 4779 DBG(dca, DWARN, "dca->fm_capabilities = 0x%x", dca->fm_capabilities); 4780 4781 /* Only register with IO Fault Services if we have some capability */ 4782 if (dca->fm_capabilities) { 4783 dca_regsattr.devacc_attr_access = DDI_FLAGERR_ACC; 4784 dca_dmaattr.dma_attr_flags = DDI_DMA_FLAGERR; 4785 4786 /* Register capabilities with IO Fault Services */ 4787 ddi_fm_init(dca->dca_dip, &dca->fm_capabilities, &fm_ibc); 4788 DBG(dca, DWARN, "fm_capable() = 0x%x", 4789 ddi_fm_capable(dca->dca_dip)); 4790 4791 /* 4792 * Initialize pci ereport capabilities if ereport capable 4793 */ 4794 if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) || 4795 DDI_FM_ERRCB_CAP(dca->fm_capabilities)) 4796 pci_ereport_setup(dca->dca_dip); 4797 4798 /* 4799 * Initialize callback mutex and register error callback if 4800 * error callback capable. 4801 */ 4802 if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) { 4803 ddi_fm_handler_register(dca->dca_dip, dca_fm_error_cb, 4804 (void *)dca); 4805 } 4806 } else { 4807 /* 4808 * These fields have to be cleared of FMA if there are no 4809 * FMA capabilities at runtime. 4810 */ 4811 dca_regsattr.devacc_attr_access = DDI_DEFAULT_ACC; 4812 dca_dmaattr.dma_attr_flags = 0; 4813 } 4814 } 4815 4816 4817 static void 4818 dca_fma_fini(dca_t *dca) 4819 { 4820 /* Only unregister FMA capabilities if we registered some */ 4821 if (dca->fm_capabilities) { 4822 4823 /* 4824 * Release any resources allocated by pci_ereport_setup() 4825 */ 4826 if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) || 4827 DDI_FM_ERRCB_CAP(dca->fm_capabilities)) { 4828 pci_ereport_teardown(dca->dca_dip); 4829 } 4830 4831 /* 4832 * Free callback mutex and un-register error callback if 4833 * error callback capable. 4834 */ 4835 if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) { 4836 ddi_fm_handler_unregister(dca->dca_dip); 4837 } 4838 4839 /* Unregister from IO Fault Services */ 4840 ddi_fm_fini(dca->dca_dip); 4841 DBG(dca, DWARN, "fm_capable() = 0x%x", 4842 ddi_fm_capable(dca->dca_dip)); 4843 } 4844 } 4845 4846 4847 /* 4848 * The IO fault service error handling callback function 4849 */ 4850 /*ARGSUSED*/ 4851 static int 4852 dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 4853 { 4854 dca_t *dca = (dca_t *)impl_data; 4855 4856 pci_ereport_post(dip, err, NULL); 4857 if (err->fme_status == DDI_FM_FATAL) { 4858 dca_failure(dca, DDI_DATAPATH_FAULT, 4859 DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR, 4860 "fault PCI in FMA callback."); 4861 } 4862 return (err->fme_status); 4863 } 4864 4865 4866 static int 4867 dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle, 4868 dca_fma_eclass_t eclass_index) 4869 { 4870 ddi_fm_error_t de; 4871 int version = 0; 4872 4873 ddi_fm_acc_err_get(handle, &de, version); 4874 if (de.fme_status != DDI_FM_OK) { 4875 dca_failure(dca, DDI_DATAPATH_FAULT, 4876 eclass_index, fm_ena_increment(de.fme_ena), 4877 CRYPTO_DEVICE_ERROR, ""); 4878 return (DDI_FAILURE); 4879 } 4880 4881 return (DDI_SUCCESS); 4882 } 4883 4884 int 4885 dca_check_dma_handle(dca_t *dca, ddi_dma_handle_t handle, 4886 dca_fma_eclass_t eclass_index) 4887 { 4888 ddi_fm_error_t de; 4889 int version = 0; 4890 4891 ddi_fm_dma_err_get(handle, &de, version); 4892 if (de.fme_status != DDI_FM_OK) { 4893 dca_failure(dca, DDI_DATAPATH_FAULT, 4894 eclass_index, fm_ena_increment(de.fme_ena), 4895 CRYPTO_DEVICE_ERROR, ""); 4896 return (DDI_FAILURE); 4897 } 4898 return (DDI_SUCCESS); 4899 } 4900 4901 static uint64_t 4902 dca_ena(uint64_t ena) 4903 { 4904 if (ena == 0) 4905 ena = fm_ena_generate(0, FM_ENA_FMT1); 4906 else 4907 ena = fm_ena_increment(ena); 4908 return (ena); 4909 } 4910 4911 static char * 4912 dca_fma_eclass_string(char *model, dca_fma_eclass_t index) 4913 { 4914 if (strstr(model, "500")) 4915 return (dca_fma_eclass_sca500[index]); 4916 else 4917 return (dca_fma_eclass_sca1000[index]); 4918 } 4919