/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Deimos - cryptographic acceleration based upon Broadcom 582x. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Core Deimos driver. */ static void dca_enlist2(dca_listnode_t *, dca_listnode_t *, kmutex_t *); static void dca_rmlist2(dca_listnode_t *node, kmutex_t *); static dca_listnode_t *dca_delist2(dca_listnode_t *q, kmutex_t *); static void dca_free_context_list(dca_t *dca); static int dca_free_context_low(crypto_ctx_t *ctx); static int dca_attach(dev_info_t *, ddi_attach_cmd_t); static int dca_detach(dev_info_t *, ddi_detach_cmd_t); static int dca_suspend(dca_t *); static int dca_resume(dca_t *); static int dca_init(dca_t *); static int dca_reset(dca_t *, int); static int dca_initworklist(dca_t *, dca_worklist_t *); static void dca_uninit(dca_t *); static void dca_initq(dca_listnode_t *); static void dca_enqueue(dca_listnode_t *, dca_listnode_t *); static dca_listnode_t *dca_dequeue(dca_listnode_t *); static dca_listnode_t *dca_unqueue(dca_listnode_t *); static dca_request_t *dca_newreq(dca_t *); static dca_work_t *dca_getwork(dca_t *, int); static void dca_freework(dca_work_t *); static dca_work_t *dca_newwork(dca_t *); static void dca_destroywork(dca_work_t *); static void dca_schedule(dca_t *, int); static void dca_reclaim(dca_t *, int); static uint_t dca_intr(char *); static void dca_failure(dca_t *, ddi_fault_location_t, dca_fma_eclass_t index, uint64_t, int, char *, ...); static void dca_jobtimeout(void *); static int dca_drain(dca_t *); static void dca_undrain(dca_t *); static void dca_rejectjobs(dca_t *); #ifdef SCHEDDELAY static void dca_schedtimeout(void *); #endif /* * We want these inlined for performance. */ #ifndef DEBUG #pragma inline(dca_freereq, dca_getreq, dca_freework, dca_getwork) #pragma inline(dca_enqueue, dca_dequeue, dca_rmqueue, dca_done) #pragma inline(dca_reverse, dca_length) #endif /* * Device operations. */ static struct dev_ops devops = { DEVO_REV, /* devo_rev */ 0, /* devo_refcnt */ nodev, /* devo_getinfo */ nulldev, /* devo_identify */ nulldev, /* devo_probe */ dca_attach, /* devo_attach */ dca_detach, /* devo_detach */ nodev, /* devo_reset */ NULL, /* devo_cb_ops */ NULL, /* devo_bus_ops */ ddi_power, /* devo_power */ ddi_quiesce_not_supported, /* devo_quiesce */ }; #define IDENT "PCI Crypto Accelerator" #define IDENT_SYM "Crypto Accel Sym 2.0" #define IDENT_ASYM "Crypto Accel Asym 2.0" /* Space-padded, will be filled in dynamically during registration */ #define IDENT3 "PCI Crypto Accelerator Mod 2.0" #define VENDOR "Sun Microsystems, Inc." #define STALETIME (30 * SECOND) #define crypto_prov_notify crypto_provider_notification /* A 28 char function name doesn't leave much line space */ /* * Module linkage. */ static struct modldrv modldrv = { &mod_driverops, /* drv_modops */ IDENT, /* drv_linkinfo */ &devops, /* drv_dev_ops */ }; extern struct mod_ops mod_cryptoops; static struct modlcrypto modlcrypto = { &mod_cryptoops, IDENT3 }; static struct modlinkage modlinkage = { MODREV_1, /* ml_rev */ &modldrv, /* ml_linkage */ &modlcrypto, NULL }; /* * CSPI information (entry points, provider info, etc.) */ /* Mechanisms for the symmetric cipher provider */ static crypto_mech_info_t dca_mech_info_tab1[] = { /* DES-CBC */ {SUN_CKM_DES_CBC, DES_CBC_MECH_INFO_TYPE, CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC, DES_KEY_LEN, DES_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES}, /* 3DES-CBC */ {SUN_CKM_DES3_CBC, DES3_CBC_MECH_INFO_TYPE, CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC, DES3_MIN_KEY_LEN, DES3_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES} }; /* Mechanisms for the asymmetric cipher provider */ static crypto_mech_info_t dca_mech_info_tab2[] = { /* DSA */ {SUN_CKM_DSA, DSA_MECH_INFO_TYPE, CRYPTO_FG_SIGN | CRYPTO_FG_VERIFY | CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_VERIFY_ATOMIC, CRYPTO_BYTES2BITS(DSA_MIN_KEY_LEN), CRYPTO_BYTES2BITS(DSA_MAX_KEY_LEN), CRYPTO_KEYSIZE_UNIT_IN_BITS}, /* RSA */ {SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE, CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC | CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC | CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC, CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN), CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN), CRYPTO_KEYSIZE_UNIT_IN_BITS}, {SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE, CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC | CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC | CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC, CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN), CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN), CRYPTO_KEYSIZE_UNIT_IN_BITS} }; static void dca_provider_status(crypto_provider_handle_t, uint_t *); static crypto_control_ops_t dca_control_ops = { dca_provider_status }; static int dca_encrypt_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static int dca_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, crypto_req_handle_t); static int dca_encrypt_update(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, crypto_req_handle_t); static int dca_encrypt_final(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t); static int dca_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static int dca_decrypt_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static int dca_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, crypto_req_handle_t); static int dca_decrypt_update(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, crypto_req_handle_t); static int dca_decrypt_final(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t); static int dca_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static crypto_cipher_ops_t dca_cipher_ops = { dca_encrypt_init, dca_encrypt, dca_encrypt_update, dca_encrypt_final, dca_encrypt_atomic, dca_decrypt_init, dca_decrypt, dca_decrypt_update, dca_decrypt_final, dca_decrypt_atomic }; static int dca_sign_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static int dca_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, crypto_req_handle_t); static int dca_sign_update(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t); static int dca_sign_final(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t); static int dca_sign_atomic(crypto_provider_handle_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static int dca_sign_recover_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static int dca_sign_recover(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, crypto_req_handle_t); static int dca_sign_recover_atomic(crypto_provider_handle_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static crypto_sign_ops_t dca_sign_ops = { dca_sign_init, dca_sign, dca_sign_update, dca_sign_final, dca_sign_atomic, dca_sign_recover_init, dca_sign_recover, dca_sign_recover_atomic }; static int dca_verify_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static int dca_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, crypto_req_handle_t); static int dca_verify_update(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t); static int dca_verify_final(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t); static int dca_verify_atomic(crypto_provider_handle_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static int dca_verify_recover_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static int dca_verify_recover(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, crypto_req_handle_t); static int dca_verify_recover_atomic(crypto_provider_handle_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static crypto_verify_ops_t dca_verify_ops = { dca_verify_init, dca_verify, dca_verify_update, dca_verify_final, dca_verify_atomic, dca_verify_recover_init, dca_verify_recover, dca_verify_recover_atomic }; static int dca_generate_random(crypto_provider_handle_t, crypto_session_id_t, uchar_t *, size_t, crypto_req_handle_t); static crypto_random_number_ops_t dca_random_number_ops = { NULL, dca_generate_random }; static int ext_info_sym(crypto_provider_handle_t prov, crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq); static int ext_info_asym(crypto_provider_handle_t prov, crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq); static int ext_info_base(crypto_provider_handle_t prov, crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id); static crypto_provider_management_ops_t dca_provmanage_ops_1 = { ext_info_sym, /* ext_info */ NULL, /* init_token */ NULL, /* init_pin */ NULL /* set_pin */ }; static crypto_provider_management_ops_t dca_provmanage_ops_2 = { ext_info_asym, /* ext_info */ NULL, /* init_token */ NULL, /* init_pin */ NULL /* set_pin */ }; int dca_free_context(crypto_ctx_t *); static crypto_ctx_ops_t dca_ctx_ops = { NULL, dca_free_context }; /* Operations for the symmetric cipher provider */ static crypto_ops_t dca_crypto_ops1 = { &dca_control_ops, NULL, /* digest_ops */ &dca_cipher_ops, NULL, /* mac_ops */ NULL, /* sign_ops */ NULL, /* verify_ops */ NULL, /* dual_ops */ NULL, /* cipher_mac_ops */ NULL, /* random_number_ops */ NULL, /* session_ops */ NULL, /* object_ops */ NULL, /* key_ops */ &dca_provmanage_ops_1, /* management_ops */ &dca_ctx_ops }; /* Operations for the asymmetric cipher provider */ static crypto_ops_t dca_crypto_ops2 = { &dca_control_ops, NULL, /* digest_ops */ &dca_cipher_ops, NULL, /* mac_ops */ &dca_sign_ops, &dca_verify_ops, NULL, /* dual_ops */ NULL, /* cipher_mac_ops */ &dca_random_number_ops, NULL, /* session_ops */ NULL, /* object_ops */ NULL, /* key_ops */ &dca_provmanage_ops_2, /* management_ops */ &dca_ctx_ops }; /* Provider information for the symmetric cipher provider */ static crypto_provider_info_t dca_prov_info1 = { CRYPTO_SPI_VERSION_1, NULL, /* pi_provider_description */ CRYPTO_HW_PROVIDER, NULL, /* pi_provider_dev */ NULL, /* pi_provider_handle */ &dca_crypto_ops1, sizeof (dca_mech_info_tab1)/sizeof (crypto_mech_info_t), dca_mech_info_tab1, 0, /* pi_logical_provider_count */ NULL /* pi_logical_providers */ }; /* Provider information for the asymmetric cipher provider */ static crypto_provider_info_t dca_prov_info2 = { CRYPTO_SPI_VERSION_1, NULL, /* pi_provider_description */ CRYPTO_HW_PROVIDER, NULL, /* pi_provider_dev */ NULL, /* pi_provider_handle */ &dca_crypto_ops2, sizeof (dca_mech_info_tab2)/sizeof (crypto_mech_info_t), dca_mech_info_tab2, 0, /* pi_logical_provider_count */ NULL /* pi_logical_providers */ }; /* Convenience macros */ #define DCA_SOFTC_FROM_CTX(ctx) ((dca_t *)(ctx)->cc_provider) #define DCA_MECH_FROM_CTX(ctx) \ (((dca_request_t *)(ctx)->cc_provider_private)->dr_ctx.ctx_cm_type) static int dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset, caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags, dca_chain_t *head, int *n_chain); static uint64_t dca_ena(uint64_t ena); static caddr_t dca_bufdaddr_out(crypto_data_t *data); static char *dca_fma_eclass_string(char *model, dca_fma_eclass_t index); static int dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle, dca_fma_eclass_t eclass_index); static void dca_fma_init(dca_t *dca); static void dca_fma_fini(dca_t *dca); static int dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data); static dca_device_t dca_devices[] = { /* Broadcom vanilla variants */ { 0x14e4, 0x5820, "Broadcom 5820" }, { 0x14e4, 0x5821, "Broadcom 5821" }, { 0x14e4, 0x5822, "Broadcom 5822" }, { 0x14e4, 0x5825, "Broadcom 5825" }, /* Sun specific OEMd variants */ { 0x108e, 0x5454, "SCA" }, { 0x108e, 0x5455, "SCA 1000" }, { 0x108e, 0x5457, "SCA 500" }, /* subsysid should be 0x5457, but got 0x1 from HW. Assume both here. */ { 0x108e, 0x1, "SCA 500" }, }; /* * Device attributes. */ static struct ddi_device_acc_attr dca_regsattr = { DDI_DEVICE_ATTR_V1, DDI_STRUCTURE_LE_ACC, DDI_STRICTORDER_ACC, DDI_FLAGERR_ACC }; static struct ddi_device_acc_attr dca_devattr = { DDI_DEVICE_ATTR_V0, DDI_STRUCTURE_LE_ACC, DDI_STRICTORDER_ACC }; #if !defined(i386) && !defined(__i386) static struct ddi_device_acc_attr dca_bufattr = { DDI_DEVICE_ATTR_V0, DDI_NEVERSWAP_ACC, DDI_STRICTORDER_ACC }; #endif static struct ddi_dma_attr dca_dmaattr = { DMA_ATTR_V0, /* dma_attr_version */ 0x0, /* dma_attr_addr_lo */ 0xffffffffUL, /* dma_attr_addr_hi */ 0x00ffffffUL, /* dma_attr_count_max */ 0x40, /* dma_attr_align */ 0x40, /* dma_attr_burstsizes */ 0x1, /* dma_attr_minxfer */ 0x00ffffffUL, /* dma_attr_maxxfer */ 0xffffffffUL, /* dma_attr_seg */ #if defined(i386) || defined(__i386) || defined(__amd64) 512, /* dma_attr_sgllen */ #else 1, /* dma_attr_sgllen */ #endif 1, /* dma_attr_granular */ DDI_DMA_FLAGERR /* dma_attr_flags */ }; static void *dca_state = NULL; int dca_mindma = 2500; /* * FMA eclass string definitions. Note that these string arrays must be * consistent with the dca_fma_eclass_t enum. */ static char *dca_fma_eclass_sca1000[] = { "sca1000.hw.device", "sca1000.hw.timeout", "sca1000.none" }; static char *dca_fma_eclass_sca500[] = { "sca500.hw.device", "sca500.hw.timeout", "sca500.none" }; /* * DDI entry points. */ int _init(void) { int rv; DBG(NULL, DMOD, "dca: in _init"); if ((rv = ddi_soft_state_init(&dca_state, sizeof (dca_t), 1)) != 0) { /* this should *never* happen! */ return (rv); } if ((rv = mod_install(&modlinkage)) != 0) { /* cleanup here */ ddi_soft_state_fini(&dca_state); return (rv); } return (0); } int _fini(void) { int rv; DBG(NULL, DMOD, "dca: in _fini"); if ((rv = mod_remove(&modlinkage)) == 0) { /* cleanup here */ ddi_soft_state_fini(&dca_state); } return (rv); } int _info(struct modinfo *modinfop) { DBG(NULL, DMOD, "dca: in _info"); return (mod_info(&modlinkage, modinfop)); } int dca_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { ddi_acc_handle_t pci; int instance; ddi_iblock_cookie_t ibc; int intr_added = 0; dca_t *dca; ushort_t venid; ushort_t devid; ushort_t revid; ushort_t subsysid; ushort_t subvenid; int i; int ret; char ID[64]; static char *unknowndev = "Unknown device"; #if DEBUG /* these are only used for debugging */ ushort_t pcicomm; ushort_t pcistat; uchar_t cachelinesz; uchar_t mingnt; uchar_t maxlat; uchar_t lattmr; #endif instance = ddi_get_instance(dip); DBG(NULL, DMOD, "dca: in dca_attach() for %d", instance); switch (cmd) { case DDI_RESUME: if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) { dca_diperror(dip, "no soft state in detach"); return (DDI_FAILURE); } /* assumption: we won't be DDI_DETACHed until we return */ return (dca_resume(dca)); case DDI_ATTACH: break; default: return (DDI_FAILURE); } if (ddi_slaveonly(dip) == DDI_SUCCESS) { dca_diperror(dip, "slot does not support PCI bus-master"); return (DDI_FAILURE); } if (ddi_intr_hilevel(dip, 0) != 0) { dca_diperror(dip, "hilevel interrupts not supported"); return (DDI_FAILURE); } if (pci_config_setup(dip, &pci) != DDI_SUCCESS) { dca_diperror(dip, "unable to setup PCI config handle"); return (DDI_FAILURE); } /* common PCI attributes */ venid = pci_config_get16(pci, PCI_VENID); devid = pci_config_get16(pci, PCI_DEVID); revid = pci_config_get8(pci, PCI_REVID); subvenid = pci_config_get16(pci, PCI_SUBVENID); subsysid = pci_config_get16(pci, PCI_SUBSYSID); /* * Broadcom-specific timings. * We disable these timers/counters since they can cause * incorrect false failures when the bus is just a little * bit slow, or busy. */ pci_config_put8(pci, PCI_TRDYTO, 0); pci_config_put8(pci, PCI_RETRIES, 0); /* initialize PCI access settings */ pci_config_put16(pci, PCI_COMM, PCICOMM_SEE | PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE); /* set up our PCI latency timer */ pci_config_put8(pci, PCI_LATTMR, 0x40); #if DEBUG /* read registers (for debugging) */ pcicomm = pci_config_get16(pci, PCI_COMM); pcistat = pci_config_get16(pci, PCI_STATUS); cachelinesz = pci_config_get8(pci, PCI_CACHELINESZ); mingnt = pci_config_get8(pci, PCI_MINGNT); maxlat = pci_config_get8(pci, PCI_MAXLAT); lattmr = pci_config_get8(pci, PCI_LATTMR); #endif pci_config_teardown(&pci); if (ddi_get_iblock_cookie(dip, 0, &ibc) != DDI_SUCCESS) { dca_diperror(dip, "unable to get iblock cookie"); return (DDI_FAILURE); } if (ddi_soft_state_zalloc(dca_state, instance) != DDI_SUCCESS) { dca_diperror(dip, "unable to allocate soft state"); return (DDI_FAILURE); } dca = ddi_get_soft_state(dca_state, instance); ASSERT(dca != NULL); dca->dca_dip = dip; WORKLIST(dca, MCR1)->dwl_prov = NULL; WORKLIST(dca, MCR2)->dwl_prov = NULL; /* figure pagesize */ dca->dca_pagesize = ddi_ptob(dip, 1); /* * Search for the device in our supported devices table. This * is here for two reasons. First, we want to ensure that * only Sun-qualified (and presumably Sun-labeled) devices can * be used with this driver. Second, some devices have * specific differences. E.g. the 5821 has support for a * special mode of RC4, deeper queues, power management, and * other changes. Also, the export versions of some of these * chips don't support RC4 or 3DES, so we catch that here. * * Note that we only look at the upper nibble of the device * id, which is used to distinguish export vs. domestic * versions of the chip. (The lower nibble is used for * stepping information.) */ for (i = 0; i < (sizeof (dca_devices) / sizeof (dca_device_t)); i++) { /* * Try to match the subsystem information first. */ if (subvenid && (subvenid == dca_devices[i].dd_vendor_id) && subsysid && (subsysid == dca_devices[i].dd_device_id)) { dca->dca_model = dca_devices[i].dd_model; dca->dca_devid = dca_devices[i].dd_device_id; break; } /* * Failing that, try the generic vendor and device id. * Even if we find a match, we keep searching anyway, * since we would prefer to find a match based on the * subsystem ids. */ if ((venid == dca_devices[i].dd_vendor_id) && (devid == dca_devices[i].dd_device_id)) { dca->dca_model = dca_devices[i].dd_model; dca->dca_devid = dca_devices[i].dd_device_id; } } /* try and handle an unrecognized device */ if (dca->dca_model == NULL) { dca->dca_model = unknowndev; dca_error(dca, "device not recognized, not supported"); DBG(dca, DPCI, "i=%d venid=%x devid=%x rev=%d", i, venid, devid, revid); } if (ddi_prop_update_string(DDI_DEV_T_NONE, dip, "description", dca->dca_model) != DDI_SUCCESS) { dca_error(dca, "unable to create description property"); return (DDI_FAILURE); } DBG(dca, DPCI, "PCI command=0x%x status=%x cachelinesz=%x", pcicomm, pcistat, cachelinesz); DBG(dca, DPCI, "mingnt=0x%x maxlat=0x%x lattmr=0x%x", mingnt, maxlat, lattmr); /* * initialize locks, etc. */ (void) mutex_init(&dca->dca_intrlock, NULL, MUTEX_DRIVER, ibc); /* use RNGSHA1 by default */ if (ddi_getprop(DDI_DEV_T_ANY, dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "rngdirect", 0) == 0) { dca->dca_flags |= DCA_RNGSHA1; } /* initialize FMA */ dca_fma_init(dca); /* initialize some key data structures */ if (dca_init(dca) != DDI_SUCCESS) { goto failed; } /* initialize kstats */ dca_ksinit(dca); /* setup access to registers */ if (ddi_regs_map_setup(dip, 1, (caddr_t *)&dca->dca_regs, 0, 0, &dca_regsattr, &dca->dca_regs_handle) != DDI_SUCCESS) { dca_error(dca, "unable to map registers"); goto failed; } DBG(dca, DCHATTY, "MCR1 = %x", GETCSR(dca, CSR_MCR1)); DBG(dca, DCHATTY, "CONTROL = %x", GETCSR(dca, CSR_DMACTL)); DBG(dca, DCHATTY, "STATUS = %x", GETCSR(dca, CSR_DMASTAT)); DBG(dca, DCHATTY, "DMAEA = %x", GETCSR(dca, CSR_DMAEA)); DBG(dca, DCHATTY, "MCR2 = %x", GETCSR(dca, CSR_MCR2)); /* reset the chip */ if (dca_reset(dca, 0) < 0) { goto failed; } /* initialize the chip */ PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64); if (dca_check_acc_handle(dca, dca->dca_regs_handle, DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { goto failed; } /* add the interrupt */ if (ddi_add_intr(dip, 0, &dca->dca_icookie, NULL, dca_intr, (void *)dca) != DDI_SUCCESS) { DBG(dca, DWARN, "ddi_add_intr failed"); goto failed; } else { intr_added = 1; } /* enable interrupts on the device */ /* * XXX: Note, 5820A1 errata indicates that this may clobber * bits 24 and 23, which affect the speed of the RNG. Since * we always want to run in full-speed mode, this should be * harmless. */ if (dca->dca_devid == 0x5825) { /* for 5825 - increase the DMA read size */ SETBIT(dca, CSR_DMACTL, DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256); } else { SETBIT(dca, CSR_DMACTL, DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE); } if (dca_check_acc_handle(dca, dca->dca_regs_handle, DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { goto failed; } /* register MCR1 with the crypto framework */ /* Be careful not to exceed 32 chars */ (void) sprintf(ID, "%s/%d %s", ddi_driver_name(dip), ddi_get_instance(dip), IDENT_SYM); dca_prov_info1.pi_provider_description = ID; dca_prov_info1.pi_provider_dev.pd_hw = dip; dca_prov_info1.pi_provider_handle = dca; if ((ret = crypto_register_provider(&dca_prov_info1, &WORKLIST(dca, MCR1)->dwl_prov)) != CRYPTO_SUCCESS) { cmn_err(CE_WARN, "crypto_register_provider() failed (%d) for MCR1", ret); goto failed; } /* register MCR2 with the crypto framework */ /* Be careful not to exceed 32 chars */ (void) sprintf(ID, "%s/%d %s", ddi_driver_name(dip), ddi_get_instance(dip), IDENT_ASYM); dca_prov_info2.pi_provider_description = ID; dca_prov_info2.pi_provider_dev.pd_hw = dip; dca_prov_info2.pi_provider_handle = dca; if ((ret = crypto_register_provider(&dca_prov_info2, &WORKLIST(dca, MCR2)->dwl_prov)) != CRYPTO_SUCCESS) { cmn_err(CE_WARN, "crypto_register_provider() failed (%d) for MCR2", ret); goto failed; } crypto_prov_notify(WORKLIST(dca, MCR1)->dwl_prov, CRYPTO_PROVIDER_READY); crypto_prov_notify(WORKLIST(dca, MCR2)->dwl_prov, CRYPTO_PROVIDER_READY); /* Initialize the local random number pool for this instance */ if ((ret = dca_random_init(dca)) != CRYPTO_SUCCESS) { goto failed; } mutex_enter(&dca->dca_intrlock); dca->dca_jobtid = timeout(dca_jobtimeout, (void *)dca, drv_usectohz(SECOND)); mutex_exit(&dca->dca_intrlock); ddi_set_driver_private(dip, (caddr_t)dca); ddi_report_dev(dip); if (ddi_get_devstate(dca->dca_dip) != DDI_DEVSTATE_UP) { ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_RESTORED); } return (DDI_SUCCESS); failed: /* unregister from the crypto framework */ if (WORKLIST(dca, MCR1)->dwl_prov != NULL) { (void) crypto_unregister_provider( WORKLIST(dca, MCR1)->dwl_prov); } if (WORKLIST(dca, MCR2)->dwl_prov != NULL) { (void) crypto_unregister_provider( WORKLIST(dca, MCR2)->dwl_prov); } if (intr_added) { CLRBIT(dca, CSR_DMACTL, DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE); /* unregister intr handler */ ddi_remove_intr(dip, 0, dca->dca_icookie); } if (dca->dca_regs_handle) { ddi_regs_map_free(&dca->dca_regs_handle); } if (dca->dca_intrstats) { kstat_delete(dca->dca_intrstats); } if (dca->dca_ksp) { kstat_delete(dca->dca_ksp); } dca_uninit(dca); /* finalize FMA */ dca_fma_fini(dca); mutex_destroy(&dca->dca_intrlock); ddi_soft_state_free(dca_state, instance); return (DDI_FAILURE); } int dca_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { int instance; dca_t *dca; timeout_id_t tid; instance = ddi_get_instance(dip); DBG(NULL, DMOD, "dca: in dca_detach() for %d", instance); switch (cmd) { case DDI_SUSPEND: if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) { dca_diperror(dip, "no soft state in detach"); return (DDI_FAILURE); } /* assumption: we won't be DDI_DETACHed until we return */ return (dca_suspend(dca)); case DDI_DETACH: break; default: return (DDI_FAILURE); } if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) { dca_diperror(dip, "no soft state in detach"); return (DDI_FAILURE); } /* * Unregister from kCF. * This needs to be done at the beginning of detach. */ if (WORKLIST(dca, MCR1)->dwl_prov != NULL) { if (crypto_unregister_provider( WORKLIST(dca, MCR1)->dwl_prov) != CRYPTO_SUCCESS) { dca_error(dca, "unable to unregister MCR1 from kcf"); return (DDI_FAILURE); } } if (WORKLIST(dca, MCR2)->dwl_prov != NULL) { if (crypto_unregister_provider( WORKLIST(dca, MCR2)->dwl_prov) != CRYPTO_SUCCESS) { dca_error(dca, "unable to unregister MCR2 from kcf"); return (DDI_FAILURE); } } /* * Cleanup the private context list. Once the * crypto_unregister_provider returns, it is safe to do so. */ dca_free_context_list(dca); /* Cleanup the local random number pool */ dca_random_fini(dca); /* send any jobs in the waitq back to kCF */ dca_rejectjobs(dca); /* untimeout the timeouts */ mutex_enter(&dca->dca_intrlock); tid = dca->dca_jobtid; dca->dca_jobtid = 0; mutex_exit(&dca->dca_intrlock); if (tid) { (void) untimeout(tid); } /* disable device interrupts */ CLRBIT(dca, CSR_DMACTL, DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE); /* unregister interrupt handlers */ ddi_remove_intr(dip, 0, dca->dca_icookie); /* release our regs handle */ ddi_regs_map_free(&dca->dca_regs_handle); /* toss out kstats */ if (dca->dca_intrstats) { kstat_delete(dca->dca_intrstats); } if (dca->dca_ksp) { kstat_delete(dca->dca_ksp); } mutex_destroy(&dca->dca_intrlock); dca_uninit(dca); /* finalize FMA */ dca_fma_fini(dca); ddi_soft_state_free(dca_state, instance); return (DDI_SUCCESS); } int dca_resume(dca_t *dca) { ddi_acc_handle_t pci; if (pci_config_setup(dca->dca_dip, &pci) != DDI_SUCCESS) { dca_error(dca, "unable to setup PCI config handle"); return (DDI_FAILURE); } /* * Reprogram registers in PCI configuration space. */ /* Broadcom-specific timers -- we disable them. */ pci_config_put8(pci, PCI_TRDYTO, 0); pci_config_put8(pci, PCI_RETRIES, 0); /* initialize PCI access settings */ pci_config_put16(pci, PCI_COMM, PCICOMM_SEE | PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE); /* set up our PCI latency timer */ pci_config_put8(pci, PCI_LATTMR, 0x40); pci_config_teardown(&pci); if (dca_reset(dca, 0) < 0) { dca_error(dca, "unable to reset device during resume"); return (DDI_FAILURE); } /* * Now restore the card-specific CSRs. */ /* restore endianness settings */ PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64); if (dca_check_acc_handle(dca, dca->dca_regs_handle, DCA_FM_ECLASS_NONE) != DDI_SUCCESS) return (DDI_FAILURE); /* restore interrupt enables */ if (dca->dca_devid == 0x5825) { /* for 5825 set 256 byte read size to improve performance */ SETBIT(dca, CSR_DMACTL, DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256); } else { SETBIT(dca, CSR_DMACTL, DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE); } if (dca_check_acc_handle(dca, dca->dca_regs_handle, DCA_FM_ECLASS_NONE) != DDI_SUCCESS) return (DDI_FAILURE); /* resume scheduling jobs on the device */ dca_undrain(dca); return (DDI_SUCCESS); } int dca_suspend(dca_t *dca) { if ((dca_drain(dca)) != 0) { return (DDI_FAILURE); } if (dca_reset(dca, 0) < 0) { dca_error(dca, "unable to reset device during suspend"); return (DDI_FAILURE); } return (DDI_SUCCESS); } /* * Hardware access stuff. */ int dca_reset(dca_t *dca, int failreset) { int i; if (dca->dca_regs_handle == NULL) { return (-1); } PUTCSR(dca, CSR_DMACTL, DMACTL_RESET); if (!failreset) { if (dca_check_acc_handle(dca, dca->dca_regs_handle, DCA_FM_ECLASS_NONE) != DDI_SUCCESS) return (-1); } /* now wait for a reset */ for (i = 1; i < 100; i++) { uint32_t dmactl; drv_usecwait(100); dmactl = GETCSR(dca, CSR_DMACTL); if (!failreset) { if (dca_check_acc_handle(dca, dca->dca_regs_handle, DCA_FM_ECLASS_NONE) != DDI_SUCCESS) return (-1); } if ((dmactl & DMACTL_RESET) == 0) { DBG(dca, DCHATTY, "reset in %d usec", i * 100); return (0); } } if (!failreset) { dca_failure(dca, DDI_DEVICE_FAULT, DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR, "timeout waiting for reset after %d usec", i * 100); } return (-1); } int dca_initworklist(dca_t *dca, dca_worklist_t *wlp) { int i; int reqprealloc = wlp->dwl_hiwater + (MAXWORK * MAXREQSPERMCR); /* * Set up work queue. */ mutex_init(&wlp->dwl_lock, NULL, MUTEX_DRIVER, dca->dca_icookie); mutex_init(&wlp->dwl_freereqslock, NULL, MUTEX_DRIVER, dca->dca_icookie); mutex_init(&wlp->dwl_freelock, NULL, MUTEX_DRIVER, dca->dca_icookie); cv_init(&wlp->dwl_cv, NULL, CV_DRIVER, NULL); mutex_enter(&wlp->dwl_lock); dca_initq(&wlp->dwl_freereqs); dca_initq(&wlp->dwl_waitq); dca_initq(&wlp->dwl_freework); dca_initq(&wlp->dwl_runq); for (i = 0; i < MAXWORK; i++) { dca_work_t *workp; if ((workp = dca_newwork(dca)) == NULL) { dca_error(dca, "unable to allocate work"); mutex_exit(&wlp->dwl_lock); return (DDI_FAILURE); } workp->dw_wlp = wlp; dca_freework(workp); } mutex_exit(&wlp->dwl_lock); for (i = 0; i < reqprealloc; i++) { dca_request_t *reqp; if ((reqp = dca_newreq(dca)) == NULL) { dca_error(dca, "unable to allocate request"); return (DDI_FAILURE); } reqp->dr_dca = dca; reqp->dr_wlp = wlp; dca_freereq(reqp); } return (DDI_SUCCESS); } int dca_init(dca_t *dca) { dca_worklist_t *wlp; /* Initialize the private context list and the corresponding lock. */ mutex_init(&dca->dca_ctx_list_lock, NULL, MUTEX_DRIVER, NULL); dca_initq(&dca->dca_ctx_list); /* * MCR1 algorithms. */ wlp = WORKLIST(dca, MCR1); (void) sprintf(wlp->dwl_name, "dca%d:mcr1", ddi_get_instance(dca->dca_dip)); wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY, dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "mcr1_lowater", MCR1LOWATER); wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY, dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "mcr1_hiwater", MCR1HIWATER); wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY, dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "mcr1_maxreqs", MCR1MAXREQS), MAXREQSPERMCR); wlp->dwl_dca = dca; wlp->dwl_mcr = MCR1; if (dca_initworklist(dca, wlp) != DDI_SUCCESS) { return (DDI_FAILURE); } /* * MCR2 algorithms. */ wlp = WORKLIST(dca, MCR2); (void) sprintf(wlp->dwl_name, "dca%d:mcr2", ddi_get_instance(dca->dca_dip)); wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY, dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "mcr2_lowater", MCR2LOWATER); wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY, dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "mcr2_hiwater", MCR2HIWATER); wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY, dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "mcr2_maxreqs", MCR2MAXREQS), MAXREQSPERMCR); wlp->dwl_dca = dca; wlp->dwl_mcr = MCR2; if (dca_initworklist(dca, wlp) != DDI_SUCCESS) { return (DDI_FAILURE); } return (DDI_SUCCESS); } /* * Uninitialize worklists. This routine should only be called when no * active jobs (hence DMA mappings) exist. One way to ensure this is * to unregister from kCF before calling this routine. (This is done * e.g. in detach(9e).) */ void dca_uninit(dca_t *dca) { int mcr; mutex_destroy(&dca->dca_ctx_list_lock); for (mcr = MCR1; mcr <= MCR2; mcr++) { dca_worklist_t *wlp = WORKLIST(dca, mcr); dca_work_t *workp; dca_request_t *reqp; if (dca->dca_regs_handle == NULL) { continue; } mutex_enter(&wlp->dwl_lock); while ((workp = dca_getwork(dca, mcr)) != NULL) { dca_destroywork(workp); } mutex_exit(&wlp->dwl_lock); while ((reqp = dca_getreq(dca, mcr, 0)) != NULL) { dca_destroyreq(reqp); } mutex_destroy(&wlp->dwl_lock); mutex_destroy(&wlp->dwl_freereqslock); mutex_destroy(&wlp->dwl_freelock); cv_destroy(&wlp->dwl_cv); wlp->dwl_prov = NULL; } } static void dca_enlist2(dca_listnode_t *q, dca_listnode_t *node, kmutex_t *lock) { if (!q || !node) return; mutex_enter(lock); node->dl_next2 = q; node->dl_prev2 = q->dl_prev2; node->dl_next2->dl_prev2 = node; node->dl_prev2->dl_next2 = node; mutex_exit(lock); } static void dca_rmlist2(dca_listnode_t *node, kmutex_t *lock) { if (!node) return; mutex_enter(lock); node->dl_next2->dl_prev2 = node->dl_prev2; node->dl_prev2->dl_next2 = node->dl_next2; node->dl_next2 = NULL; node->dl_prev2 = NULL; mutex_exit(lock); } static dca_listnode_t * dca_delist2(dca_listnode_t *q, kmutex_t *lock) { dca_listnode_t *node; mutex_enter(lock); if ((node = q->dl_next2) == q) { mutex_exit(lock); return (NULL); } node->dl_next2->dl_prev2 = node->dl_prev2; node->dl_prev2->dl_next2 = node->dl_next2; node->dl_next2 = NULL; node->dl_prev2 = NULL; mutex_exit(lock); return (node); } void dca_initq(dca_listnode_t *q) { q->dl_next = q; q->dl_prev = q; q->dl_next2 = q; q->dl_prev2 = q; } void dca_enqueue(dca_listnode_t *q, dca_listnode_t *node) { /* * Enqueue submits at the "tail" of the list, i.e. just * behind the sentinel. */ node->dl_next = q; node->dl_prev = q->dl_prev; node->dl_next->dl_prev = node; node->dl_prev->dl_next = node; } void dca_rmqueue(dca_listnode_t *node) { node->dl_next->dl_prev = node->dl_prev; node->dl_prev->dl_next = node->dl_next; node->dl_next = NULL; node->dl_prev = NULL; } dca_listnode_t * dca_dequeue(dca_listnode_t *q) { dca_listnode_t *node; /* * Dequeue takes from the "head" of the list, i.e. just after * the sentinel. */ if ((node = q->dl_next) == q) { /* queue is empty */ return (NULL); } dca_rmqueue(node); return (node); } /* this is the opposite of dequeue, it takes things off in LIFO order */ dca_listnode_t * dca_unqueue(dca_listnode_t *q) { dca_listnode_t *node; /* * unqueue takes from the "tail" of the list, i.e. just before * the sentinel. */ if ((node = q->dl_prev) == q) { /* queue is empty */ return (NULL); } dca_rmqueue(node); return (node); } dca_listnode_t * dca_peekqueue(dca_listnode_t *q) { dca_listnode_t *node; if ((node = q->dl_next) == q) { return (NULL); } else { return (node); } } /* * Interrupt service routine. */ uint_t dca_intr(char *arg) { dca_t *dca = (dca_t *)arg; uint32_t status; mutex_enter(&dca->dca_intrlock); status = GETCSR(dca, CSR_DMASTAT); PUTCSR(dca, CSR_DMASTAT, status & DMASTAT_INTERRUPTS); if (dca_check_acc_handle(dca, dca->dca_regs_handle, DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { mutex_exit(&dca->dca_intrlock); return ((uint_t)DDI_FAILURE); } DBG(dca, DINTR, "interrupted, status = 0x%x!", status); if ((status & DMASTAT_INTERRUPTS) == 0) { /* increment spurious interrupt kstat */ if (dca->dca_intrstats) { KIOIP(dca)->intrs[KSTAT_INTR_SPURIOUS]++; } mutex_exit(&dca->dca_intrlock); return (DDI_INTR_UNCLAIMED); } if (dca->dca_intrstats) { KIOIP(dca)->intrs[KSTAT_INTR_HARD]++; } if (status & DMASTAT_MCR1INT) { DBG(dca, DINTR, "MCR1 interrupted"); mutex_enter(&(WORKLIST(dca, MCR1)->dwl_lock)); dca_schedule(dca, MCR1); dca_reclaim(dca, MCR1); mutex_exit(&(WORKLIST(dca, MCR1)->dwl_lock)); } if (status & DMASTAT_MCR2INT) { DBG(dca, DINTR, "MCR2 interrupted"); mutex_enter(&(WORKLIST(dca, MCR2)->dwl_lock)); dca_schedule(dca, MCR2); dca_reclaim(dca, MCR2); mutex_exit(&(WORKLIST(dca, MCR2)->dwl_lock)); } if (status & DMASTAT_ERRINT) { uint32_t erraddr; erraddr = GETCSR(dca, CSR_DMAEA); mutex_exit(&dca->dca_intrlock); /* * bit 1 of the error address indicates failure during * read if set, during write otherwise. */ dca_failure(dca, DDI_DEVICE_FAULT, DCA_FM_ECLASS_HW_DEVICE, dca_ena(0), CRYPTO_DEVICE_ERROR, "DMA master access error %s address 0x%x", erraddr & 0x1 ? "reading" : "writing", erraddr & ~1); return (DDI_INTR_CLAIMED); } mutex_exit(&dca->dca_intrlock); return (DDI_INTR_CLAIMED); } /* * Reverse a string of bytes from s1 into s2. The reversal happens * from the tail of s1. If len1 < len2, then null bytes will be * padded to the end of s2. If len2 < len1, then (presumably null) * bytes will be dropped from the start of s1. * * The rationale here is that when s1 (source) is shorter, then we * are reversing from big-endian ordering, into device ordering, and * want to add some extra nulls to the tail (MSB) side of the device. * * Similarly, when s2 (dest) is shorter, then we are truncating what * are presumably null MSB bits from the device. * * There is an expectation when reversing from the device back into * big-endian, that the number of bytes to reverse and the target size * will match, and no truncation or padding occurs. */ void dca_reverse(void *s1, void *s2, int len1, int len2) { caddr_t src, dst; if (len1 == 0) { if (len2) { bzero(s2, len2); } return; } src = (caddr_t)s1 + len1 - 1; dst = s2; while ((src >= (caddr_t)s1) && (len2)) { *dst++ = *src--; len2--; } while (len2 > 0) { *dst++ = 0; len2--; } } uint16_t dca_padfull(int num) { if (num <= 512) { return (BITS2BYTES(512)); } if (num <= 768) { return (BITS2BYTES(768)); } if (num <= 1024) { return (BITS2BYTES(1024)); } if (num <= 1536) { return (BITS2BYTES(1536)); } if (num <= 2048) { return (BITS2BYTES(2048)); } return (0); } uint16_t dca_padhalf(int num) { if (num <= 256) { return (BITS2BYTES(256)); } if (num <= 384) { return (BITS2BYTES(384)); } if (num <= 512) { return (BITS2BYTES(512)); } if (num <= 768) { return (BITS2BYTES(768)); } if (num <= 1024) { return (BITS2BYTES(1024)); } return (0); } dca_work_t * dca_newwork(dca_t *dca) { dca_work_t *workp; size_t size; ddi_dma_cookie_t c; unsigned nc; int rv; workp = kmem_zalloc(sizeof (dca_work_t), KM_SLEEP); rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, DDI_DMA_SLEEP, NULL, &workp->dw_mcr_dmah); if (rv != 0) { dca_error(dca, "unable to alloc MCR DMA handle"); dca_destroywork(workp); return (NULL); } rv = ddi_dma_mem_alloc(workp->dw_mcr_dmah, ROUNDUP(MCR_SIZE, dca->dca_pagesize), &dca_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &workp->dw_mcr_kaddr, &size, &workp->dw_mcr_acch); if (rv != 0) { dca_error(dca, "unable to alloc MCR DMA memory"); dca_destroywork(workp); return (NULL); } rv = ddi_dma_addr_bind_handle(workp->dw_mcr_dmah, NULL, workp->dw_mcr_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_RDWR, DDI_DMA_SLEEP, NULL, &c, &nc); if (rv != DDI_DMA_MAPPED) { dca_error(dca, "unable to map MCR DMA memory"); dca_destroywork(workp); return (NULL); } workp->dw_mcr_paddr = c.dmac_address; return (workp); } void dca_destroywork(dca_work_t *workp) { if (workp->dw_mcr_paddr) { (void) ddi_dma_unbind_handle(workp->dw_mcr_dmah); } if (workp->dw_mcr_acch) { ddi_dma_mem_free(&workp->dw_mcr_acch); } if (workp->dw_mcr_dmah) { ddi_dma_free_handle(&workp->dw_mcr_dmah); } kmem_free(workp, sizeof (dca_work_t)); } dca_request_t * dca_newreq(dca_t *dca) { dca_request_t *reqp; size_t size; ddi_dma_cookie_t c; unsigned nc; int rv; int n_chain = 0; size = (DESC_SIZE * MAXFRAGS) + CTX_MAXLENGTH; reqp = kmem_zalloc(sizeof (dca_request_t), KM_SLEEP); reqp->dr_dca = dca; /* * Setup the DMA region for the context and descriptors. */ rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, DDI_DMA_SLEEP, NULL, &reqp->dr_ctx_dmah); if (rv != DDI_SUCCESS) { dca_error(dca, "failure allocating request DMA handle"); dca_destroyreq(reqp); return (NULL); } /* for driver hardening, allocate in whole pages */ rv = ddi_dma_mem_alloc(reqp->dr_ctx_dmah, ROUNDUP(size, dca->dca_pagesize), &dca_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &reqp->dr_ctx_kaddr, &size, &reqp->dr_ctx_acch); if (rv != DDI_SUCCESS) { dca_error(dca, "unable to alloc request DMA memory"); dca_destroyreq(reqp); return (NULL); } rv = ddi_dma_addr_bind_handle(reqp->dr_ctx_dmah, NULL, reqp->dr_ctx_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_WRITE, DDI_DMA_SLEEP, 0, &c, &nc); if (rv != DDI_DMA_MAPPED) { dca_error(dca, "failed binding request DMA handle"); dca_destroyreq(reqp); return (NULL); } reqp->dr_ctx_paddr = c.dmac_address; reqp->dr_dma_size = size; /* * Set up the dma for our scratch/shared buffers. */ rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_dmah); if (rv != DDI_SUCCESS) { dca_error(dca, "failure allocating ibuf DMA handle"); dca_destroyreq(reqp); return (NULL); } rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_dmah); if (rv != DDI_SUCCESS) { dca_error(dca, "failure allocating obuf DMA handle"); dca_destroyreq(reqp); return (NULL); } rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, DDI_DMA_SLEEP, NULL, &reqp->dr_chain_in_dmah); if (rv != DDI_SUCCESS) { dca_error(dca, "failure allocating chain_in DMA handle"); dca_destroyreq(reqp); return (NULL); } rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, DDI_DMA_SLEEP, NULL, &reqp->dr_chain_out_dmah); if (rv != DDI_SUCCESS) { dca_error(dca, "failure allocating chain_out DMA handle"); dca_destroyreq(reqp); return (NULL); } /* * for driver hardening, allocate in whole pages. */ size = ROUNDUP(MAXPACKET, dca->dca_pagesize); #if defined(i386) || defined(__i386) /* * Use kmem_alloc instead of ddi_dma_mem_alloc here since the latter * may fail on x86 platform if a physically contiguous memory chunk * cannot be found. From initial testing, we did not see performance * degradation as seen on Sparc. */ if ((reqp->dr_ibuf_kaddr = kmem_alloc(size, KM_SLEEP)) == NULL) { dca_error(dca, "unable to alloc request ibuf memory"); dca_destroyreq(reqp); return (NULL); } if ((reqp->dr_obuf_kaddr = kmem_alloc(size, KM_SLEEP)) == NULL) { dca_error(dca, "unable to alloc request obuf memory"); dca_destroyreq(reqp); return (NULL); } #else /* * We could kmem_alloc for Sparc too. However, it gives worse * performance when transferring more than one page data. For example, * using 4 threads and 12032 byte data and 3DES on 900MHZ Sparc system, * kmem_alloc uses 80% CPU and ddi_dma_mem_alloc uses 50% CPU for * the same throughput. */ rv = ddi_dma_mem_alloc(reqp->dr_ibuf_dmah, size, &dca_bufattr, DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_kaddr, &size, &reqp->dr_ibuf_acch); if (rv != DDI_SUCCESS) { dca_error(dca, "unable to alloc request DMA memory"); dca_destroyreq(reqp); return (NULL); } rv = ddi_dma_mem_alloc(reqp->dr_obuf_dmah, size, &dca_bufattr, DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_kaddr, &size, &reqp->dr_obuf_acch); if (rv != DDI_SUCCESS) { dca_error(dca, "unable to alloc request DMA memory"); dca_destroyreq(reqp); return (NULL); } #endif /* Skip the used portion in the context page */ reqp->dr_offset = CTX_MAXLENGTH; if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset, reqp->dr_ibuf_kaddr, reqp->dr_ibuf_dmah, DDI_DMA_WRITE | DDI_DMA_STREAMING, &reqp->dr_ibuf_head, &n_chain)) != DDI_SUCCESS) { (void) dca_destroyreq(reqp); return (NULL); } reqp->dr_ibuf_paddr = reqp->dr_ibuf_head.dc_buffer_paddr; /* Skip the space used by the input buffer */ reqp->dr_offset += DESC_SIZE * n_chain; if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset, reqp->dr_obuf_kaddr, reqp->dr_obuf_dmah, DDI_DMA_READ | DDI_DMA_STREAMING, &reqp->dr_obuf_head, &n_chain)) != DDI_SUCCESS) { (void) dca_destroyreq(reqp); return (NULL); } reqp->dr_obuf_paddr = reqp->dr_obuf_head.dc_buffer_paddr; /* Skip the space used by the output buffer */ reqp->dr_offset += DESC_SIZE * n_chain; DBG(dca, DCHATTY, "CTX is 0x%p, phys 0x%x, len %d", reqp->dr_ctx_kaddr, reqp->dr_ctx_paddr, CTX_MAXLENGTH); return (reqp); } void dca_destroyreq(dca_request_t *reqp) { #if defined(i386) || defined(__i386) dca_t *dca = reqp->dr_dca; size_t size = ROUNDUP(MAXPACKET, dca->dca_pagesize); #endif /* * Clean up DMA for the context structure. */ if (reqp->dr_ctx_paddr) { (void) ddi_dma_unbind_handle(reqp->dr_ctx_dmah); } if (reqp->dr_ctx_acch) { ddi_dma_mem_free(&reqp->dr_ctx_acch); } if (reqp->dr_ctx_dmah) { ddi_dma_free_handle(&reqp->dr_ctx_dmah); } /* * Clean up DMA for the scratch buffer. */ #if defined(i386) || defined(__i386) if (reqp->dr_ibuf_dmah) { (void) ddi_dma_unbind_handle(reqp->dr_ibuf_dmah); ddi_dma_free_handle(&reqp->dr_ibuf_dmah); } if (reqp->dr_obuf_dmah) { (void) ddi_dma_unbind_handle(reqp->dr_obuf_dmah); ddi_dma_free_handle(&reqp->dr_obuf_dmah); } kmem_free(reqp->dr_ibuf_kaddr, size); kmem_free(reqp->dr_obuf_kaddr, size); #else if (reqp->dr_ibuf_paddr) { (void) ddi_dma_unbind_handle(reqp->dr_ibuf_dmah); } if (reqp->dr_obuf_paddr) { (void) ddi_dma_unbind_handle(reqp->dr_obuf_dmah); } if (reqp->dr_ibuf_acch) { ddi_dma_mem_free(&reqp->dr_ibuf_acch); } if (reqp->dr_obuf_acch) { ddi_dma_mem_free(&reqp->dr_obuf_acch); } if (reqp->dr_ibuf_dmah) { ddi_dma_free_handle(&reqp->dr_ibuf_dmah); } if (reqp->dr_obuf_dmah) { ddi_dma_free_handle(&reqp->dr_obuf_dmah); } #endif /* * These two DMA handles should have been unbinded in * dca_unbindchains() function */ if (reqp->dr_chain_in_dmah) { ddi_dma_free_handle(&reqp->dr_chain_in_dmah); } if (reqp->dr_chain_out_dmah) { ddi_dma_free_handle(&reqp->dr_chain_out_dmah); } kmem_free(reqp, sizeof (dca_request_t)); } dca_work_t * dca_getwork(dca_t *dca, int mcr) { dca_worklist_t *wlp = WORKLIST(dca, mcr); dca_work_t *workp; mutex_enter(&wlp->dwl_freelock); workp = (dca_work_t *)dca_dequeue(&wlp->dwl_freework); mutex_exit(&wlp->dwl_freelock); if (workp) { int nreqs; bzero(workp->dw_mcr_kaddr, 8); /* clear out old requests */ for (nreqs = 0; nreqs < MAXREQSPERMCR; nreqs++) { workp->dw_reqs[nreqs] = NULL; } } return (workp); } void dca_freework(dca_work_t *workp) { mutex_enter(&workp->dw_wlp->dwl_freelock); dca_enqueue(&workp->dw_wlp->dwl_freework, (dca_listnode_t *)workp); mutex_exit(&workp->dw_wlp->dwl_freelock); } dca_request_t * dca_getreq(dca_t *dca, int mcr, int tryhard) { dca_worklist_t *wlp = WORKLIST(dca, mcr); dca_request_t *reqp; mutex_enter(&wlp->dwl_freereqslock); reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_freereqs); mutex_exit(&wlp->dwl_freereqslock); if (reqp) { reqp->dr_flags = 0; reqp->dr_callback = NULL; } else if (tryhard) { /* * failed to get a free one, try an allocation, the hard way. * XXX: Kstat desired here. */ if ((reqp = dca_newreq(dca)) != NULL) { reqp->dr_wlp = wlp; reqp->dr_dca = dca; reqp->dr_flags = 0; reqp->dr_callback = NULL; } } return (reqp); } void dca_freereq(dca_request_t *reqp) { reqp->dr_kcf_req = NULL; if (!(reqp->dr_flags & DR_NOCACHE)) { mutex_enter(&reqp->dr_wlp->dwl_freereqslock); dca_enqueue(&reqp->dr_wlp->dwl_freereqs, (dca_listnode_t *)reqp); mutex_exit(&reqp->dr_wlp->dwl_freereqslock); } } /* * Binds user buffers to DMA handles dynamically. On Sparc, a user buffer * is mapped to a single physical address. On x86, a user buffer is mapped * to multiple physical addresses. These physical addresses are chained * using the method specified in Broadcom BCM5820 specification. */ int dca_bindchains(dca_request_t *reqp, size_t incnt, size_t outcnt) { int rv; caddr_t kaddr; uint_t flags; int n_chain = 0; if (reqp->dr_flags & DR_INPLACE) { flags = DDI_DMA_RDWR | DDI_DMA_CONSISTENT; } else { flags = DDI_DMA_WRITE | DDI_DMA_STREAMING; } /* first the input */ if (incnt) { if ((kaddr = dca_bufdaddr(reqp->dr_in)) == NULL) { DBG(NULL, DWARN, "unrecognised crypto data format"); return (DDI_FAILURE); } if ((rv = dca_bindchains_one(reqp, incnt, reqp->dr_offset, kaddr, reqp->dr_chain_in_dmah, flags, &reqp->dr_chain_in_head, &n_chain)) != DDI_SUCCESS) { (void) dca_unbindchains(reqp); return (rv); } /* * The offset and length are altered by the calling routine * reqp->dr_in->cd_offset += incnt; * reqp->dr_in->cd_length -= incnt; */ /* Save the first one in the chain for MCR */ reqp->dr_in_paddr = reqp->dr_chain_in_head.dc_buffer_paddr; reqp->dr_in_next = reqp->dr_chain_in_head.dc_next_paddr; reqp->dr_in_len = reqp->dr_chain_in_head.dc_buffer_length; } else { reqp->dr_in_paddr = NULL; reqp->dr_in_next = 0; reqp->dr_in_len = 0; } if (reqp->dr_flags & DR_INPLACE) { reqp->dr_out_paddr = reqp->dr_in_paddr; reqp->dr_out_len = reqp->dr_in_len; reqp->dr_out_next = reqp->dr_in_next; return (DDI_SUCCESS); } /* then the output */ if (outcnt) { flags = DDI_DMA_READ | DDI_DMA_STREAMING; if ((kaddr = dca_bufdaddr_out(reqp->dr_out)) == NULL) { DBG(NULL, DWARN, "unrecognised crypto data format"); (void) dca_unbindchains(reqp); return (DDI_FAILURE); } rv = dca_bindchains_one(reqp, outcnt, reqp->dr_offset + n_chain * DESC_SIZE, kaddr, reqp->dr_chain_out_dmah, flags, &reqp->dr_chain_out_head, &n_chain); if (rv != DDI_SUCCESS) { (void) dca_unbindchains(reqp); return (DDI_FAILURE); } /* Save the first one in the chain for MCR */ reqp->dr_out_paddr = reqp->dr_chain_out_head.dc_buffer_paddr; reqp->dr_out_next = reqp->dr_chain_out_head.dc_next_paddr; reqp->dr_out_len = reqp->dr_chain_out_head.dc_buffer_length; } else { reqp->dr_out_paddr = NULL; reqp->dr_out_next = 0; reqp->dr_out_len = 0; } return (DDI_SUCCESS); } /* * Unbind the user buffers from the DMA handles. */ int dca_unbindchains(dca_request_t *reqp) { int rv = DDI_SUCCESS; int rv1 = DDI_SUCCESS; /* Clear the input chain */ if (reqp->dr_chain_in_head.dc_buffer_paddr != NULL) { (void) ddi_dma_unbind_handle(reqp->dr_chain_in_dmah); reqp->dr_chain_in_head.dc_buffer_paddr = 0; } if (reqp->dr_flags & DR_INPLACE) { return (rv); } /* Clear the output chain */ if (reqp->dr_chain_out_head.dc_buffer_paddr != NULL) { (void) ddi_dma_unbind_handle(reqp->dr_chain_out_dmah); reqp->dr_chain_out_head.dc_buffer_paddr = 0; } return ((rv != DDI_SUCCESS)? rv : rv1); } /* * Build either input chain or output chain. It is single-item chain for Sparc, * and possible mutiple-item chain for x86. */ static int dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset, caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags, dca_chain_t *head, int *n_chain) { ddi_dma_cookie_t c; uint_t nc; int rv; caddr_t chain_kaddr_pre; caddr_t chain_kaddr; uint32_t chain_paddr; int i; /* Advance past the context structure to the starting address */ chain_paddr = reqp->dr_ctx_paddr + dr_offset; chain_kaddr = reqp->dr_ctx_kaddr + dr_offset; /* * Bind the kernel address to the DMA handle. On x86, the actual * buffer is mapped into multiple physical addresses. On Sparc, * the actual buffer is mapped into a single address. */ rv = ddi_dma_addr_bind_handle(handle, NULL, kaddr, cnt, flags, DDI_DMA_DONTWAIT, NULL, &c, &nc); if (rv != DDI_DMA_MAPPED) { return (DDI_FAILURE); } (void) ddi_dma_sync(handle, 0, cnt, DDI_DMA_SYNC_FORDEV); if ((rv = dca_check_dma_handle(reqp->dr_dca, handle, DCA_FM_ECLASS_NONE)) != DDI_SUCCESS) { reqp->destroy = TRUE; return (rv); } *n_chain = nc; /* Setup the data buffer chain for DMA transfer */ chain_kaddr_pre = NULL; head->dc_buffer_paddr = 0; head->dc_next_paddr = 0; head->dc_buffer_length = 0; for (i = 0; i < nc; i++) { /* PIO */ PUTDESC32(reqp, chain_kaddr, DESC_BUFADDR, c.dmac_address); PUTDESC16(reqp, chain_kaddr, DESC_RSVD, 0); PUTDESC16(reqp, chain_kaddr, DESC_LENGTH, c.dmac_size); /* Remember the head of the chain */ if (head->dc_buffer_paddr == 0) { head->dc_buffer_paddr = c.dmac_address; head->dc_buffer_length = c.dmac_size; } /* Link to the previous one if one exists */ if (chain_kaddr_pre) { PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT, chain_paddr); if (head->dc_next_paddr == 0) head->dc_next_paddr = chain_paddr; } chain_kaddr_pre = chain_kaddr; /* Maintain pointers */ chain_paddr += DESC_SIZE; chain_kaddr += DESC_SIZE; /* Retrieve the next cookie if there is one */ if (i < nc-1) ddi_dma_nextcookie(handle, &c); } /* Set the next pointer in the last entry to NULL */ PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT, 0); return (DDI_SUCCESS); } /* * Schedule some work. */ int dca_start(dca_t *dca, dca_request_t *reqp, int mcr, int dosched) { dca_worklist_t *wlp = WORKLIST(dca, mcr); mutex_enter(&wlp->dwl_lock); DBG(dca, DCHATTY, "req=%p, in=%p, out=%p, ctx=%p, ibuf=%p, obuf=%p", reqp, reqp->dr_in, reqp->dr_out, reqp->dr_ctx_kaddr, reqp->dr_ibuf_kaddr, reqp->dr_obuf_kaddr); DBG(dca, DCHATTY, "ctx paddr = %x, ibuf paddr = %x, obuf paddr = %x", reqp->dr_ctx_paddr, reqp->dr_ibuf_paddr, reqp->dr_obuf_paddr); /* sync out the entire context and descriptor chains */ (void) ddi_dma_sync(reqp->dr_ctx_dmah, 0, 0, DDI_DMA_SYNC_FORDEV); if (dca_check_dma_handle(dca, reqp->dr_ctx_dmah, DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { reqp->destroy = TRUE; mutex_exit(&wlp->dwl_lock); return (CRYPTO_DEVICE_ERROR); } dca_enqueue(&wlp->dwl_waitq, (dca_listnode_t *)reqp); wlp->dwl_count++; wlp->dwl_lastsubmit = ddi_get_lbolt(); reqp->dr_wlp = wlp; if ((wlp->dwl_count == wlp->dwl_hiwater) && (wlp->dwl_busy == 0)) { /* we are fully loaded now, let kCF know */ wlp->dwl_flowctl++; wlp->dwl_busy = 1; crypto_prov_notify(wlp->dwl_prov, CRYPTO_PROVIDER_BUSY); } if (dosched) { #ifdef SCHEDDELAY /* possibly wait for more work to arrive */ if (wlp->dwl_count >= wlp->dwl_reqspermcr) { dca_schedule(dca, mcr); } else if (!wlp->dwl_schedtid) { /* wait 1 msec for more work before doing it */ wlp->dwl_schedtid = timeout(dca_schedtimeout, (void *)wlp, drv_usectohz(MSEC)); } #else dca_schedule(dca, mcr); #endif } mutex_exit(&wlp->dwl_lock); return (CRYPTO_QUEUED); } void dca_schedule(dca_t *dca, int mcr) { dca_worklist_t *wlp = WORKLIST(dca, mcr); int csr; int full; uint32_t status; ASSERT(mutex_owned(&wlp->dwl_lock)); /* * If the card is draining or has an outstanding failure, * don't schedule any more work on it right now */ if (wlp->dwl_drain || (dca->dca_flags & DCA_FAILED)) { return; } if (mcr == MCR2) { csr = CSR_MCR2; full = DMASTAT_MCR2FULL; } else { csr = CSR_MCR1; full = DMASTAT_MCR1FULL; } for (;;) { dca_work_t *workp; uint32_t offset; int nreqs; status = GETCSR(dca, CSR_DMASTAT); if (dca_check_acc_handle(dca, dca->dca_regs_handle, DCA_FM_ECLASS_NONE) != DDI_SUCCESS) return; if ((status & full) != 0) break; #ifdef SCHEDDELAY /* if there isn't enough to do, don't bother now */ if ((wlp->dwl_count < wlp->dwl_reqspermcr) && (ddi_get_lbolt() < (wlp->dwl_lastsubmit + drv_usectohz(MSEC)))) { /* wait a bit longer... */ if (wlp->dwl_schedtid == 0) { wlp->dwl_schedtid = timeout(dca_schedtimeout, (void *)wlp, drv_usectohz(MSEC)); } return; } #endif /* grab a work structure */ workp = dca_getwork(dca, mcr); if (workp == NULL) { /* * There must be work ready to be reclaimed, * in this case, since the chip can only hold * less work outstanding than there are total. */ dca_reclaim(dca, mcr); continue; } nreqs = 0; offset = MCR_CTXADDR; while (nreqs < wlp->dwl_reqspermcr) { dca_request_t *reqp; reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_waitq); if (reqp == NULL) { /* nothing left to process */ break; } /* * Update flow control. */ wlp->dwl_count--; if ((wlp->dwl_count == wlp->dwl_lowater) && (wlp->dwl_busy)) { wlp->dwl_busy = 0; crypto_prov_notify(wlp->dwl_prov, CRYPTO_PROVIDER_READY); } /* * Context address. */ PUTMCR32(workp, offset, reqp->dr_ctx_paddr); offset += 4; /* * Input chain. */ /* input buffer address */ PUTMCR32(workp, offset, reqp->dr_in_paddr); offset += 4; /* next input buffer entry */ PUTMCR32(workp, offset, reqp->dr_in_next); offset += 4; /* input buffer length */ PUTMCR16(workp, offset, reqp->dr_in_len); offset += 2; /* zero the reserved field */ PUTMCR16(workp, offset, 0); offset += 2; /* * Overall length. */ /* reserved field */ PUTMCR16(workp, offset, 0); offset += 2; /* total packet length */ PUTMCR16(workp, offset, reqp->dr_pkt_length); offset += 2; /* * Output chain. */ /* output buffer address */ PUTMCR32(workp, offset, reqp->dr_out_paddr); offset += 4; /* next output buffer entry */ PUTMCR32(workp, offset, reqp->dr_out_next); offset += 4; /* output buffer length */ PUTMCR16(workp, offset, reqp->dr_out_len); offset += 2; /* zero the reserved field */ PUTMCR16(workp, offset, 0); offset += 2; /* * Note submission. */ workp->dw_reqs[nreqs] = reqp; nreqs++; } if (nreqs == 0) { /* nothing in the queue! */ dca_freework(workp); return; } wlp->dwl_submit++; PUTMCR16(workp, MCR_FLAGS, 0); PUTMCR16(workp, MCR_COUNT, nreqs); DBG(dca, DCHATTY, "posting work (phys %x, virt 0x%p) (%d reqs) to MCR%d", workp->dw_mcr_paddr, workp->dw_mcr_kaddr, nreqs, mcr); workp->dw_lbolt = ddi_get_lbolt(); /* Make sure MCR is synced out to device. */ (void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 0, DDI_DMA_SYNC_FORDEV); if (dca_check_dma_handle(dca, workp->dw_mcr_dmah, DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { dca_destroywork(workp); return; } PUTCSR(dca, csr, workp->dw_mcr_paddr); if (dca_check_acc_handle(dca, dca->dca_regs_handle, DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { dca_destroywork(workp); return; } else { dca_enqueue(&wlp->dwl_runq, (dca_listnode_t *)workp); } DBG(dca, DCHATTY, "posted"); } } /* * Reclaim completed work, called in interrupt context. */ void dca_reclaim(dca_t *dca, int mcr) { dca_worklist_t *wlp = WORKLIST(dca, mcr); dca_work_t *workp; ushort_t flags; int nreclaimed = 0; int i; DBG(dca, DRECLAIM, "worklist = 0x%p (MCR%d)", wlp, mcr); ASSERT(mutex_owned(&wlp->dwl_lock)); /* * For each MCR in the submitted (runq), we check to see if * it has been processed. If so, then we note each individual * job in the MCR, and and do the completion processing for * each of such job. */ for (;;) { workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq); if (workp == NULL) { break; } /* only sync the MCR flags, since that's all we need */ (void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 4, DDI_DMA_SYNC_FORKERNEL); if (dca_check_dma_handle(dca, workp->dw_mcr_dmah, DCA_FM_ECLASS_NONE) != DDI_SUCCESS) { dca_rmqueue((dca_listnode_t *)workp); dca_destroywork(workp); return; } flags = GETMCR16(workp, MCR_FLAGS); if ((flags & MCRFLAG_FINISHED) == 0) { /* chip is still working on it */ DBG(dca, DRECLAIM, "chip still working on it (MCR%d)", mcr); break; } /* its really for us, so remove it from the queue */ dca_rmqueue((dca_listnode_t *)workp); /* if we were draining, signal on the cv */ if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) { cv_signal(&wlp->dwl_cv); } /* update statistics, done under the lock */ for (i = 0; i < wlp->dwl_reqspermcr; i++) { dca_request_t *reqp = workp->dw_reqs[i]; if (reqp == NULL) { continue; } if (reqp->dr_byte_stat >= 0) { dca->dca_stats[reqp->dr_byte_stat] += reqp->dr_pkt_length; } if (reqp->dr_job_stat >= 0) { dca->dca_stats[reqp->dr_job_stat]++; } } mutex_exit(&wlp->dwl_lock); for (i = 0; i < wlp->dwl_reqspermcr; i++) { dca_request_t *reqp = workp->dw_reqs[i]; if (reqp == NULL) { continue; } /* Do the callback. */ workp->dw_reqs[i] = NULL; dca_done(reqp, CRYPTO_SUCCESS); nreclaimed++; } /* now we can release the work */ dca_freework(workp); mutex_enter(&wlp->dwl_lock); } DBG(dca, DRECLAIM, "reclaimed %d cmds", nreclaimed); } int dca_length(crypto_data_t *cdata) { return (cdata->cd_length); } /* * This is the callback function called from the interrupt when a kCF job * completes. It does some driver-specific things, and then calls the * kCF-provided callback. Finally, it cleans up the state for the work * request and drops the reference count to allow for DR. */ void dca_done(dca_request_t *reqp, int err) { uint64_t ena = 0; /* unbind any chains we were using */ if (dca_unbindchains(reqp) != DDI_SUCCESS) { /* DMA failure */ ena = dca_ena(ena); dca_failure(reqp->dr_dca, DDI_DATAPATH_FAULT, DCA_FM_ECLASS_NONE, ena, CRYPTO_DEVICE_ERROR, "fault on buffer DMA handle"); if (err == CRYPTO_SUCCESS) { err = CRYPTO_DEVICE_ERROR; } } if (reqp->dr_callback != NULL) { reqp->dr_callback(reqp, err); } else { dca_freereq(reqp); } } /* * Call this when a failure is detected. It will reset the chip, * log a message, alert kCF, and mark jobs in the runq as failed. */ /* ARGSUSED */ void dca_failure(dca_t *dca, ddi_fault_location_t loc, dca_fma_eclass_t index, uint64_t ena, int errno, char *mess, ...) { va_list ap; char buf[256]; int mcr; char *eclass; int have_mutex; va_start(ap, mess); (void) vsprintf(buf, mess, ap); va_end(ap); eclass = dca_fma_eclass_string(dca->dca_model, index); if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) && index != DCA_FM_ECLASS_NONE) { ddi_fm_ereport_post(dca->dca_dip, eclass, ena, DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); /* Report the impact of the failure to the DDI. */ ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_LOST); } else { /* Just log the error string to the message log */ dca_error(dca, buf); } /* * Indicate a failure (keeps schedule from running). */ dca->dca_flags |= DCA_FAILED; /* * Reset the chip. This should also have as a side effect, the * disabling of all interrupts from the device. */ (void) dca_reset(dca, 1); /* * Report the failure to kCF. */ for (mcr = MCR1; mcr <= MCR2; mcr++) { if (WORKLIST(dca, mcr)->dwl_prov) { crypto_prov_notify(WORKLIST(dca, mcr)->dwl_prov, CRYPTO_PROVIDER_FAILED); } } /* * Return jobs not sent to hardware back to kCF. */ dca_rejectjobs(dca); /* * From this point on, no new work should be arriving, and the * chip should not be doing any active DMA. */ /* * Now find all the work submitted to the device and fail * them. */ for (mcr = MCR1; mcr <= MCR2; mcr++) { dca_worklist_t *wlp; int i; wlp = WORKLIST(dca, mcr); if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) { continue; } for (;;) { dca_work_t *workp; have_mutex = mutex_tryenter(&wlp->dwl_lock); workp = (dca_work_t *)dca_dequeue(&wlp->dwl_runq); if (workp == NULL) { if (have_mutex) mutex_exit(&wlp->dwl_lock); break; } mutex_exit(&wlp->dwl_lock); /* * Free up requests */ for (i = 0; i < wlp->dwl_reqspermcr; i++) { dca_request_t *reqp = workp->dw_reqs[i]; if (reqp) { dca_done(reqp, errno); workp->dw_reqs[i] = NULL; } } mutex_enter(&wlp->dwl_lock); /* * If waiting to drain, signal on the waiter. */ if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) { cv_signal(&wlp->dwl_cv); } /* * Return the work and request structures to * the free pool. */ dca_freework(workp); if (have_mutex) mutex_exit(&wlp->dwl_lock); } } } #ifdef SCHEDDELAY /* * Reschedule worklist as needed. */ void dca_schedtimeout(void *arg) { dca_worklist_t *wlp = (dca_worklist_t *)arg; mutex_enter(&wlp->dwl_lock); wlp->dwl_schedtid = 0; dca_schedule(wlp->dwl_dca, wlp->dwl_mcr); mutex_exit(&wlp->dwl_lock); } #endif /* * Check for stalled jobs. */ void dca_jobtimeout(void *arg) { int mcr; dca_t *dca = (dca_t *)arg; int hung = 0; for (mcr = MCR1; mcr <= MCR2; mcr++) { dca_worklist_t *wlp = WORKLIST(dca, mcr); dca_work_t *workp; clock_t when; mutex_enter(&wlp->dwl_lock); when = ddi_get_lbolt(); workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq); if (workp == NULL) { /* nothing sitting in the queue */ mutex_exit(&wlp->dwl_lock); continue; } if ((when - workp->dw_lbolt) < drv_usectohz(STALETIME)) { /* request has been queued for less than STALETIME */ mutex_exit(&wlp->dwl_lock); continue; } /* job has been sitting around for over 1 second, badness */ DBG(dca, DWARN, "stale job (0x%p) found in MCR%d!", workp, mcr); /* put it back in the queue, until we reset the chip */ hung++; mutex_exit(&wlp->dwl_lock); } if (hung) { dca_failure(dca, DDI_DEVICE_FAULT, DCA_FM_ECLASS_HW_TIMEOUT, dca_ena(0), CRYPTO_DEVICE_ERROR, "timeout processing job.)"); } /* reschedule ourself */ mutex_enter(&dca->dca_intrlock); if (dca->dca_jobtid == 0) { /* timeout has been canceled, prior to DR */ mutex_exit(&dca->dca_intrlock); return; } /* check again in 1 second */ dca->dca_jobtid = timeout(dca_jobtimeout, arg, drv_usectohz(SECOND)); mutex_exit(&dca->dca_intrlock); } /* * This returns all jobs back to kCF. It assumes that processing * on the worklist has halted. */ void dca_rejectjobs(dca_t *dca) { int mcr; int have_mutex; for (mcr = MCR1; mcr <= MCR2; mcr++) { dca_worklist_t *wlp = WORKLIST(dca, mcr); dca_request_t *reqp; if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) { continue; } have_mutex = mutex_tryenter(&wlp->dwl_lock); for (;;) { reqp = (dca_request_t *)dca_unqueue(&wlp->dwl_waitq); if (reqp == NULL) { break; } /* update flow control */ wlp->dwl_count--; if ((wlp->dwl_count == wlp->dwl_lowater) && (wlp->dwl_busy)) { wlp->dwl_busy = 0; crypto_prov_notify(wlp->dwl_prov, CRYPTO_PROVIDER_READY); } mutex_exit(&wlp->dwl_lock); (void) dca_unbindchains(reqp); reqp->dr_callback(reqp, EAGAIN); mutex_enter(&wlp->dwl_lock); } if (have_mutex) mutex_exit(&wlp->dwl_lock); } } int dca_drain(dca_t *dca) { int mcr; for (mcr = MCR1; mcr <= MCR2; mcr++) { #ifdef SCHEDDELAY timeout_id_t tid; #endif dca_worklist_t *wlp = WORKLIST(dca, mcr); mutex_enter(&wlp->dwl_lock); wlp->dwl_drain = 1; /* give it up to a second to drain from the chip */ if (!QEMPTY(&wlp->dwl_runq)) { (void) cv_reltimedwait(&wlp->dwl_cv, &wlp->dwl_lock, drv_usectohz(STALETIME), TR_CLOCK_TICK); if (!QEMPTY(&wlp->dwl_runq)) { dca_error(dca, "unable to drain device"); mutex_exit(&wlp->dwl_lock); dca_undrain(dca); return (EBUSY); } } #ifdef SCHEDDELAY tid = wlp->dwl_schedtid; mutex_exit(&wlp->dwl_lock); /* * untimeout outside the lock -- this is safe because we * have set the drain flag, so dca_schedule() will not * reschedule another timeout */ if (tid) { untimeout(tid); } #else mutex_exit(&wlp->dwl_lock); #endif } return (0); } void dca_undrain(dca_t *dca) { int mcr; for (mcr = MCR1; mcr <= MCR2; mcr++) { dca_worklist_t *wlp = WORKLIST(dca, mcr); mutex_enter(&wlp->dwl_lock); wlp->dwl_drain = 0; dca_schedule(dca, mcr); mutex_exit(&wlp->dwl_lock); } } /* * Duplicate the crypto_data_t structure, but point to the original * buffers. */ int dca_dupcrypto(crypto_data_t *input, crypto_data_t *ninput) { ninput->cd_format = input->cd_format; ninput->cd_offset = input->cd_offset; ninput->cd_length = input->cd_length; ninput->cd_miscdata = input->cd_miscdata; switch (input->cd_format) { case CRYPTO_DATA_RAW: ninput->cd_raw.iov_base = input->cd_raw.iov_base; ninput->cd_raw.iov_len = input->cd_raw.iov_len; break; case CRYPTO_DATA_UIO: ninput->cd_uio = input->cd_uio; break; case CRYPTO_DATA_MBLK: ninput->cd_mp = input->cd_mp; break; default: DBG(NULL, DWARN, "dca_dupcrypto: unrecognised crypto data format"); return (CRYPTO_FAILED); } return (CRYPTO_SUCCESS); } /* * Performs validation checks on the input and output data structures. */ int dca_verifyio(crypto_data_t *input, crypto_data_t *output) { int rv = CRYPTO_SUCCESS; switch (input->cd_format) { case CRYPTO_DATA_RAW: break; case CRYPTO_DATA_UIO: /* we support only kernel buffer */ if (input->cd_uio->uio_segflg != UIO_SYSSPACE) { DBG(NULL, DWARN, "non kernel input uio buffer"); rv = CRYPTO_ARGUMENTS_BAD; } break; case CRYPTO_DATA_MBLK: break; default: DBG(NULL, DWARN, "unrecognised input crypto data format"); rv = CRYPTO_ARGUMENTS_BAD; } switch (output->cd_format) { case CRYPTO_DATA_RAW: break; case CRYPTO_DATA_UIO: /* we support only kernel buffer */ if (output->cd_uio->uio_segflg != UIO_SYSSPACE) { DBG(NULL, DWARN, "non kernel output uio buffer"); rv = CRYPTO_ARGUMENTS_BAD; } break; case CRYPTO_DATA_MBLK: break; default: DBG(NULL, DWARN, "unrecognised output crypto data format"); rv = CRYPTO_ARGUMENTS_BAD; } return (rv); } /* * data: source crypto_data_t struct * off: offset into the source before commencing copy * count: the amount of data to copy * dest: destination buffer */ int dca_getbufbytes(crypto_data_t *data, size_t off, int count, uchar_t *dest) { int rv = CRYPTO_SUCCESS; uio_t *uiop; uint_t vec_idx; size_t cur_len; mblk_t *mp; if (count == 0) { /* We don't want anything so we're done. */ return (rv); } /* * Sanity check that we haven't specified a length greater than the * offset adjusted size of the buffer. */ if (count > (data->cd_length - off)) { return (CRYPTO_DATA_LEN_RANGE); } /* Add the internal crypto_data offset to the requested offset. */ off += data->cd_offset; switch (data->cd_format) { case CRYPTO_DATA_RAW: bcopy(data->cd_raw.iov_base + off, dest, count); break; case CRYPTO_DATA_UIO: /* * Jump to the first iovec containing data to be * processed. */ uiop = data->cd_uio; for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && off >= uiop->uio_iov[vec_idx].iov_len; off -= uiop->uio_iov[vec_idx++].iov_len) ; if (vec_idx == uiop->uio_iovcnt) { /* * The caller specified an offset that is larger than * the total size of the buffers it provided. */ return (CRYPTO_DATA_LEN_RANGE); } /* * Now process the iovecs. */ while (vec_idx < uiop->uio_iovcnt && count > 0) { cur_len = min(uiop->uio_iov[vec_idx].iov_len - off, count); bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest, cur_len); count -= cur_len; dest += cur_len; vec_idx++; off = 0; } if (vec_idx == uiop->uio_iovcnt && count > 0) { /* * The end of the specified iovec's was reached but * the length requested could not be processed * (requested to digest more data than it provided). */ return (CRYPTO_DATA_LEN_RANGE); } break; case CRYPTO_DATA_MBLK: /* * Jump to the first mblk_t containing data to be processed. */ for (mp = data->cd_mp; mp != NULL && off >= MBLKL(mp); off -= MBLKL(mp), mp = mp->b_cont) ; if (mp == NULL) { /* * The caller specified an offset that is larger than * the total size of the buffers it provided. */ return (CRYPTO_DATA_LEN_RANGE); } /* * Now do the processing on the mblk chain. */ while (mp != NULL && count > 0) { cur_len = min(MBLKL(mp) - off, count); bcopy((char *)(mp->b_rptr + off), dest, cur_len); count -= cur_len; dest += cur_len; mp = mp->b_cont; off = 0; } if (mp == NULL && count > 0) { /* * The end of the mblk was reached but the length * requested could not be processed, (requested to * digest more data than it provided). */ return (CRYPTO_DATA_LEN_RANGE); } break; default: DBG(NULL, DWARN, "unrecognised crypto data format"); rv = CRYPTO_ARGUMENTS_BAD; } return (rv); } /* * Performs the input, output or hard scatter/gather checks on the specified * crypto_data_t struct. Returns true if the data is scatter/gather in nature * ie fails the test. */ int dca_sgcheck(dca_t *dca, crypto_data_t *data, dca_sg_param_t val) { uio_t *uiop; mblk_t *mp; int rv = FALSE; switch (val) { case DCA_SG_CONTIG: /* * Check for a contiguous data buffer. */ switch (data->cd_format) { case CRYPTO_DATA_RAW: /* Contiguous in nature */ break; case CRYPTO_DATA_UIO: if (data->cd_uio->uio_iovcnt > 1) rv = TRUE; break; case CRYPTO_DATA_MBLK: mp = data->cd_mp; if (mp->b_cont != NULL) rv = TRUE; break; default: DBG(NULL, DWARN, "unrecognised crypto data format"); } break; case DCA_SG_WALIGN: /* * Check for a contiguous data buffer that is 32-bit word * aligned and is of word multiples in size. */ switch (data->cd_format) { case CRYPTO_DATA_RAW: if ((data->cd_raw.iov_len % sizeof (uint32_t)) || ((uintptr_t)data->cd_raw.iov_base % sizeof (uint32_t))) { rv = TRUE; } break; case CRYPTO_DATA_UIO: uiop = data->cd_uio; if (uiop->uio_iovcnt > 1) { return (TRUE); } /* So there is only one iovec */ if ((uiop->uio_iov[0].iov_len % sizeof (uint32_t)) || ((uintptr_t)uiop->uio_iov[0].iov_base % sizeof (uint32_t))) { rv = TRUE; } break; case CRYPTO_DATA_MBLK: mp = data->cd_mp; if (mp->b_cont != NULL) { return (TRUE); } /* So there is only one mblk in the chain */ if ((MBLKL(mp) % sizeof (uint32_t)) || ((uintptr_t)mp->b_rptr % sizeof (uint32_t))) { rv = TRUE; } break; default: DBG(NULL, DWARN, "unrecognised crypto data format"); } break; case DCA_SG_PALIGN: /* * Check that the data buffer is page aligned and is of * page multiples in size. */ switch (data->cd_format) { case CRYPTO_DATA_RAW: if ((data->cd_length % dca->dca_pagesize) || ((uintptr_t)data->cd_raw.iov_base % dca->dca_pagesize)) { rv = TRUE; } break; case CRYPTO_DATA_UIO: uiop = data->cd_uio; if ((uiop->uio_iov[0].iov_len % dca->dca_pagesize) || ((uintptr_t)uiop->uio_iov[0].iov_base % dca->dca_pagesize)) { rv = TRUE; } break; case CRYPTO_DATA_MBLK: mp = data->cd_mp; if ((MBLKL(mp) % dca->dca_pagesize) || ((uintptr_t)mp->b_rptr % dca->dca_pagesize)) { rv = TRUE; } break; default: DBG(NULL, DWARN, "unrecognised crypto data format"); } break; default: DBG(NULL, DWARN, "unrecognised scatter/gather param type"); } return (rv); } /* * Increments the cd_offset and decrements the cd_length as the data is * gathered from the crypto_data_t struct. * The data is reverse-copied into the dest buffer if the flag is true. */ int dca_gather(crypto_data_t *in, char *dest, int count, int reverse) { int rv = CRYPTO_SUCCESS; uint_t vec_idx; uio_t *uiop; off_t off = in->cd_offset; size_t cur_len; mblk_t *mp; switch (in->cd_format) { case CRYPTO_DATA_RAW: if (count > in->cd_length) { /* * The caller specified a length greater than the * size of the buffer. */ return (CRYPTO_DATA_LEN_RANGE); } if (reverse) dca_reverse(in->cd_raw.iov_base + off, dest, count, count); else bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count); in->cd_offset += count; in->cd_length -= count; break; case CRYPTO_DATA_UIO: /* * Jump to the first iovec containing data to be processed. */ uiop = in->cd_uio; for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && off >= uiop->uio_iov[vec_idx].iov_len; off -= uiop->uio_iov[vec_idx++].iov_len) ; if (vec_idx == uiop->uio_iovcnt) { /* * The caller specified an offset that is larger than * the total size of the buffers it provided. */ return (CRYPTO_DATA_LEN_RANGE); } /* * Now process the iovecs. */ while (vec_idx < uiop->uio_iovcnt && count > 0) { cur_len = min(uiop->uio_iov[vec_idx].iov_len - off, count); count -= cur_len; if (reverse) { /* Fill the dest buffer from the end */ dca_reverse(uiop->uio_iov[vec_idx].iov_base + off, dest+count, cur_len, cur_len); } else { bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest, cur_len); dest += cur_len; } in->cd_offset += cur_len; in->cd_length -= cur_len; vec_idx++; off = 0; } if (vec_idx == uiop->uio_iovcnt && count > 0) { /* * The end of the specified iovec's was reached but * the length requested could not be processed * (requested to digest more data than it provided). */ return (CRYPTO_DATA_LEN_RANGE); } break; case CRYPTO_DATA_MBLK: /* * Jump to the first mblk_t containing data to be processed. */ for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp); off -= MBLKL(mp), mp = mp->b_cont) ; if (mp == NULL) { /* * The caller specified an offset that is larger than * the total size of the buffers it provided. */ return (CRYPTO_DATA_LEN_RANGE); } /* * Now do the processing on the mblk chain. */ while (mp != NULL && count > 0) { cur_len = min(MBLKL(mp) - off, count); count -= cur_len; if (reverse) { /* Fill the dest buffer from the end */ dca_reverse((char *)(mp->b_rptr + off), dest+count, cur_len, cur_len); } else { bcopy((char *)(mp->b_rptr + off), dest, cur_len); dest += cur_len; } in->cd_offset += cur_len; in->cd_length -= cur_len; mp = mp->b_cont; off = 0; } if (mp == NULL && count > 0) { /* * The end of the mblk was reached but the length * requested could not be processed, (requested to * digest more data than it provided). */ return (CRYPTO_DATA_LEN_RANGE); } break; default: DBG(NULL, DWARN, "dca_gather: unrecognised crypto data format"); rv = CRYPTO_ARGUMENTS_BAD; } return (rv); } /* * Increments the cd_offset and decrements the cd_length as the data is * gathered from the crypto_data_t struct. */ int dca_resid_gather(crypto_data_t *in, char *resid, int *residlen, char *dest, int count) { int rv = CRYPTO_SUCCESS; caddr_t baddr; uint_t vec_idx; uio_t *uiop; off_t off = in->cd_offset; size_t cur_len; mblk_t *mp; /* Process the residual first */ if (*residlen > 0) { uint_t num = min(count, *residlen); bcopy(resid, dest, num); *residlen -= num; if (*residlen > 0) { /* * Requested amount 'count' is less than what's in * the residual, so shuffle any remaining resid to * the front. */ baddr = resid + num; bcopy(baddr, resid, *residlen); } dest += num; count -= num; } /* Now process what's in the crypto_data_t structs */ switch (in->cd_format) { case CRYPTO_DATA_RAW: if (count > in->cd_length) { /* * The caller specified a length greater than the * size of the buffer. */ return (CRYPTO_DATA_LEN_RANGE); } bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count); in->cd_offset += count; in->cd_length -= count; break; case CRYPTO_DATA_UIO: /* * Jump to the first iovec containing data to be processed. */ uiop = in->cd_uio; for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && off >= uiop->uio_iov[vec_idx].iov_len; off -= uiop->uio_iov[vec_idx++].iov_len) ; if (vec_idx == uiop->uio_iovcnt) { /* * The caller specified an offset that is larger than * the total size of the buffers it provided. */ return (CRYPTO_DATA_LEN_RANGE); } /* * Now process the iovecs. */ while (vec_idx < uiop->uio_iovcnt && count > 0) { cur_len = min(uiop->uio_iov[vec_idx].iov_len - off, count); bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest, cur_len); count -= cur_len; dest += cur_len; in->cd_offset += cur_len; in->cd_length -= cur_len; vec_idx++; off = 0; } if (vec_idx == uiop->uio_iovcnt && count > 0) { /* * The end of the specified iovec's was reached but * the length requested could not be processed * (requested to digest more data than it provided). */ return (CRYPTO_DATA_LEN_RANGE); } break; case CRYPTO_DATA_MBLK: /* * Jump to the first mblk_t containing data to be processed. */ for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp); off -= MBLKL(mp), mp = mp->b_cont) ; if (mp == NULL) { /* * The caller specified an offset that is larger than * the total size of the buffers it provided. */ return (CRYPTO_DATA_LEN_RANGE); } /* * Now do the processing on the mblk chain. */ while (mp != NULL && count > 0) { cur_len = min(MBLKL(mp) - off, count); bcopy((char *)(mp->b_rptr + off), dest, cur_len); count -= cur_len; dest += cur_len; in->cd_offset += cur_len; in->cd_length -= cur_len; mp = mp->b_cont; off = 0; } if (mp == NULL && count > 0) { /* * The end of the mblk was reached but the length * requested could not be processed, (requested to * digest more data than it provided). */ return (CRYPTO_DATA_LEN_RANGE); } break; default: DBG(NULL, DWARN, "dca_resid_gather: unrecognised crypto data format"); rv = CRYPTO_ARGUMENTS_BAD; } return (rv); } /* * Appends the data to the crypto_data_t struct increasing cd_length. * cd_offset is left unchanged. * Data is reverse-copied if the flag is TRUE. */ int dca_scatter(const char *src, crypto_data_t *out, int count, int reverse) { int rv = CRYPTO_SUCCESS; off_t offset = out->cd_offset + out->cd_length; uint_t vec_idx; uio_t *uiop; size_t cur_len; mblk_t *mp; switch (out->cd_format) { case CRYPTO_DATA_RAW: if (out->cd_raw.iov_len - offset < count) { /* Trying to write out more than space available. */ return (CRYPTO_DATA_LEN_RANGE); } if (reverse) dca_reverse((void*) src, out->cd_raw.iov_base + offset, count, count); else bcopy(src, out->cd_raw.iov_base + offset, count); out->cd_length += count; break; case CRYPTO_DATA_UIO: /* * Jump to the first iovec that can be written to. */ uiop = out->cd_uio; for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && offset >= uiop->uio_iov[vec_idx].iov_len; offset -= uiop->uio_iov[vec_idx++].iov_len) ; if (vec_idx == uiop->uio_iovcnt) { /* * The caller specified an offset that is larger than * the total size of the buffers it provided. */ return (CRYPTO_DATA_LEN_RANGE); } /* * Now process the iovecs. */ while (vec_idx < uiop->uio_iovcnt && count > 0) { cur_len = min(uiop->uio_iov[vec_idx].iov_len - offset, count); count -= cur_len; if (reverse) { dca_reverse((void*) (src+count), uiop->uio_iov[vec_idx].iov_base + offset, cur_len, cur_len); } else { bcopy(src, uiop->uio_iov[vec_idx].iov_base + offset, cur_len); src += cur_len; } out->cd_length += cur_len; vec_idx++; offset = 0; } if (vec_idx == uiop->uio_iovcnt && count > 0) { /* * The end of the specified iovec's was reached but * the length requested could not be processed * (requested to write more data than space provided). */ return (CRYPTO_DATA_LEN_RANGE); } break; case CRYPTO_DATA_MBLK: /* * Jump to the first mblk_t that can be written to. */ for (mp = out->cd_mp; mp != NULL && offset >= MBLKL(mp); offset -= MBLKL(mp), mp = mp->b_cont) ; if (mp == NULL) { /* * The caller specified an offset that is larger than * the total size of the buffers it provided. */ return (CRYPTO_DATA_LEN_RANGE); } /* * Now do the processing on the mblk chain. */ while (mp != NULL && count > 0) { cur_len = min(MBLKL(mp) - offset, count); count -= cur_len; if (reverse) { dca_reverse((void*) (src+count), (char *)(mp->b_rptr + offset), cur_len, cur_len); } else { bcopy(src, (char *)(mp->b_rptr + offset), cur_len); src += cur_len; } out->cd_length += cur_len; mp = mp->b_cont; offset = 0; } if (mp == NULL && count > 0) { /* * The end of the mblk was reached but the length * requested could not be processed, (requested to * digest more data than it provided). */ return (CRYPTO_DATA_LEN_RANGE); } break; default: DBG(NULL, DWARN, "unrecognised crypto data format"); rv = CRYPTO_ARGUMENTS_BAD; } return (rv); } /* * Compare two byte arrays in reverse order. * Return 0 if they are identical, 1 otherwise. */ int dca_bcmp_reverse(const void *s1, const void *s2, size_t n) { int i; caddr_t src, dst; if (!n) return (0); src = ((caddr_t)s1) + n - 1; dst = (caddr_t)s2; for (i = 0; i < n; i++) { if (*src != *dst) return (1); src--; dst++; } return (0); } /* * This calculates the size of a bignum in bits, specifically not counting * leading zero bits. This size calculation must be done *before* any * endian reversal takes place (i.e. the numbers are in absolute big-endian * order.) */ int dca_bitlen(unsigned char *bignum, int bytelen) { unsigned char msbyte; int i, j; for (i = 0; i < bytelen - 1; i++) { if (bignum[i] != 0) { break; } } msbyte = bignum[i]; for (j = 8; j > 1; j--) { if (msbyte & 0x80) { break; } msbyte <<= 1; } return ((8 * (bytelen - i - 1)) + j); } /* * This compares to bignums (in big-endian order). It ignores leading * null bytes. The result semantics follow bcmp, mempcmp, strcmp, etc. */ int dca_numcmp(caddr_t n1, int n1len, caddr_t n2, int n2len) { while ((n1len > 1) && (*n1 == 0)) { n1len--; n1++; } while ((n2len > 1) && (*n2 == 0)) { n2len--; n2++; } if (n1len != n2len) { return (n1len - n2len); } while ((n1len > 1) && (*n1 == *n2)) { n1++; n2++; n1len--; } return ((int)(*(uchar_t *)n1) - (int)(*(uchar_t *)n2)); } /* * Return array of key attributes. */ crypto_object_attribute_t * dca_get_key_attr(crypto_key_t *key) { if ((key->ck_format != CRYPTO_KEY_ATTR_LIST) || (key->ck_count == 0)) { return (NULL); } return (key->ck_attrs); } /* * If attribute type exists valp points to it's 32-bit value. */ int dca_attr_lookup_uint32(crypto_object_attribute_t *attrp, uint_t atnum, uint64_t atype, uint32_t *valp) { crypto_object_attribute_t *bap; bap = dca_find_attribute(attrp, atnum, atype); if (bap == NULL) { return (CRYPTO_ATTRIBUTE_TYPE_INVALID); } *valp = *bap->oa_value; return (CRYPTO_SUCCESS); } /* * If attribute type exists data contains the start address of the value, * and numelems contains it's length. */ int dca_attr_lookup_uint8_array(crypto_object_attribute_t *attrp, uint_t atnum, uint64_t atype, void **data, unsigned int *numelems) { crypto_object_attribute_t *bap; bap = dca_find_attribute(attrp, atnum, atype); if (bap == NULL) { return (CRYPTO_ATTRIBUTE_TYPE_INVALID); } *data = bap->oa_value; *numelems = bap->oa_value_len; return (CRYPTO_SUCCESS); } /* * Finds entry of specified name. If it is not found dca_find_attribute returns * NULL. */ crypto_object_attribute_t * dca_find_attribute(crypto_object_attribute_t *attrp, uint_t atnum, uint64_t atype) { while (atnum) { if (attrp->oa_type == atype) return (attrp); atnum--; attrp++; } return (NULL); } /* * Return the address of the first data buffer. If the data format is * unrecognised return NULL. */ caddr_t dca_bufdaddr(crypto_data_t *data) { switch (data->cd_format) { case CRYPTO_DATA_RAW: return (data->cd_raw.iov_base + data->cd_offset); case CRYPTO_DATA_UIO: return (data->cd_uio->uio_iov[0].iov_base + data->cd_offset); case CRYPTO_DATA_MBLK: return ((char *)data->cd_mp->b_rptr + data->cd_offset); default: DBG(NULL, DWARN, "dca_bufdaddr: unrecognised crypto data format"); return (NULL); } } static caddr_t dca_bufdaddr_out(crypto_data_t *data) { size_t offset = data->cd_offset + data->cd_length; switch (data->cd_format) { case CRYPTO_DATA_RAW: return (data->cd_raw.iov_base + offset); case CRYPTO_DATA_UIO: return (data->cd_uio->uio_iov[0].iov_base + offset); case CRYPTO_DATA_MBLK: return ((char *)data->cd_mp->b_rptr + offset); default: DBG(NULL, DWARN, "dca_bufdaddr_out: unrecognised crypto data format"); return (NULL); } } /* * Control entry points. */ /* ARGSUSED */ static void dca_provider_status(crypto_provider_handle_t provider, uint_t *status) { *status = CRYPTO_PROVIDER_READY; } /* * Cipher (encrypt/decrypt) entry points. */ /* ARGSUSED */ static int dca_encrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_encrypt_init: started"); /* check mechanism */ switch (mechanism->cm_type) { case DES_CBC_MECH_INFO_TYPE: error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP, DR_ENCRYPT); break; case DES3_CBC_MECH_INFO_TYPE: error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP, DR_ENCRYPT | DR_TRIPLE); break; case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); break; default: cmn_err(CE_WARN, "dca_encrypt_init: unexpected mech type " "0x%llx\n", (unsigned long long)mechanism->cm_type); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_encrypt_init: done, err = 0x%x", error); if (error == CRYPTO_SUCCESS) dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, &softc->dca_ctx_list_lock); return (error); } /* ARGSUSED */ static int dca_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext, crypto_data_t *ciphertext, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_encrypt: started"); /* handle inplace ops */ if (!ciphertext) { dca_request_t *reqp = ctx->cc_provider_private; reqp->dr_flags |= DR_INPLACE; ciphertext = plaintext; } /* check mechanism */ switch (DCA_MECH_FROM_CTX(ctx)) { case DES_CBC_MECH_INFO_TYPE: error = dca_3des(ctx, plaintext, ciphertext, req, DR_ENCRYPT); break; case DES3_CBC_MECH_INFO_TYPE: error = dca_3des(ctx, plaintext, ciphertext, req, DR_ENCRYPT | DR_TRIPLE); break; case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsastart(ctx, plaintext, ciphertext, req, DCA_RSA_ENC); break; default: /* Should never reach here */ cmn_err(CE_WARN, "dca_encrypt: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); error = CRYPTO_MECHANISM_INVALID; } if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) && (error != CRYPTO_BUFFER_TOO_SMALL)) { ciphertext->cd_length = 0; } DBG(softc, DENTRY, "dca_encrypt: done, err = 0x%x", error); return (error); } /* ARGSUSED */ static int dca_encrypt_update(crypto_ctx_t *ctx, crypto_data_t *plaintext, crypto_data_t *ciphertext, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_encrypt_update: started"); /* handle inplace ops */ if (!ciphertext) { dca_request_t *reqp = ctx->cc_provider_private; reqp->dr_flags |= DR_INPLACE; ciphertext = plaintext; } /* check mechanism */ switch (DCA_MECH_FROM_CTX(ctx)) { case DES_CBC_MECH_INFO_TYPE: error = dca_3desupdate(ctx, plaintext, ciphertext, req, DR_ENCRYPT); break; case DES3_CBC_MECH_INFO_TYPE: error = dca_3desupdate(ctx, plaintext, ciphertext, req, DR_ENCRYPT | DR_TRIPLE); break; default: /* Should never reach here */ cmn_err(CE_WARN, "dca_encrypt_update: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_encrypt_update: done, err = 0x%x", error); return (error); } /* ARGSUSED */ static int dca_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *ciphertext, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_encrypt_final: started"); /* check mechanism */ switch (DCA_MECH_FROM_CTX(ctx)) { case DES_CBC_MECH_INFO_TYPE: error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT); break; case DES3_CBC_MECH_INFO_TYPE: error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT | DR_TRIPLE); break; default: /* Should never reach here */ cmn_err(CE_WARN, "dca_encrypt_final: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_encrypt_final: done, err = 0x%x", error); return (error); } /* ARGSUSED */ static int dca_encrypt_atomic(crypto_provider_handle_t provider, crypto_session_id_t session_id, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext, crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc = (dca_t *)provider; DBG(softc, DENTRY, "dca_encrypt_atomic: started"); if (ctx_template != NULL) return (CRYPTO_ARGUMENTS_BAD); /* handle inplace ops */ if (!ciphertext) { ciphertext = plaintext; } /* check mechanism */ switch (mechanism->cm_type) { case DES_CBC_MECH_INFO_TYPE: error = dca_3desatomic(provider, session_id, mechanism, key, plaintext, ciphertext, KM_SLEEP, req, DR_ENCRYPT | DR_ATOMIC); break; case DES3_CBC_MECH_INFO_TYPE: error = dca_3desatomic(provider, session_id, mechanism, key, plaintext, ciphertext, KM_SLEEP, req, DR_ENCRYPT | DR_TRIPLE | DR_ATOMIC); break; case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsaatomic(provider, session_id, mechanism, key, plaintext, ciphertext, KM_SLEEP, req, DCA_RSA_ENC); break; default: cmn_err(CE_WARN, "dca_encrypt_atomic: unexpected mech type " "0x%llx\n", (unsigned long long)mechanism->cm_type); error = CRYPTO_MECHANISM_INVALID; } if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) { ciphertext->cd_length = 0; } DBG(softc, DENTRY, "dca_encrypt_atomic: done, err = 0x%x", error); return (error); } /* ARGSUSED */ static int dca_decrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_decrypt_init: started"); /* check mechanism */ switch (mechanism->cm_type) { case DES_CBC_MECH_INFO_TYPE: error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP, DR_DECRYPT); break; case DES3_CBC_MECH_INFO_TYPE: error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP, DR_DECRYPT | DR_TRIPLE); break; case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); break; default: cmn_err(CE_WARN, "dca_decrypt_init: unexpected mech type " "0x%llx\n", (unsigned long long)mechanism->cm_type); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_decrypt_init: done, err = 0x%x", error); if (error == CRYPTO_SUCCESS) dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, &softc->dca_ctx_list_lock); return (error); } /* ARGSUSED */ static int dca_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext, crypto_data_t *plaintext, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_decrypt: started"); /* handle inplace ops */ if (!plaintext) { dca_request_t *reqp = ctx->cc_provider_private; reqp->dr_flags |= DR_INPLACE; plaintext = ciphertext; } /* check mechanism */ switch (DCA_MECH_FROM_CTX(ctx)) { case DES_CBC_MECH_INFO_TYPE: error = dca_3des(ctx, ciphertext, plaintext, req, DR_DECRYPT); break; case DES3_CBC_MECH_INFO_TYPE: error = dca_3des(ctx, ciphertext, plaintext, req, DR_DECRYPT | DR_TRIPLE); break; case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsastart(ctx, ciphertext, plaintext, req, DCA_RSA_DEC); break; default: /* Should never reach here */ cmn_err(CE_WARN, "dca_decrypt: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); error = CRYPTO_MECHANISM_INVALID; } if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) && (error != CRYPTO_BUFFER_TOO_SMALL)) { if (plaintext) plaintext->cd_length = 0; } DBG(softc, DENTRY, "dca_decrypt: done, err = 0x%x", error); return (error); } /* ARGSUSED */ static int dca_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext, crypto_data_t *plaintext, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_decrypt_update: started"); /* handle inplace ops */ if (!plaintext) { dca_request_t *reqp = ctx->cc_provider_private; reqp->dr_flags |= DR_INPLACE; plaintext = ciphertext; } /* check mechanism */ switch (DCA_MECH_FROM_CTX(ctx)) { case DES_CBC_MECH_INFO_TYPE: error = dca_3desupdate(ctx, ciphertext, plaintext, req, DR_DECRYPT); break; case DES3_CBC_MECH_INFO_TYPE: error = dca_3desupdate(ctx, ciphertext, plaintext, req, DR_DECRYPT | DR_TRIPLE); break; default: /* Should never reach here */ cmn_err(CE_WARN, "dca_decrypt_update: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_decrypt_update: done, err = 0x%x", error); return (error); } /* ARGSUSED */ static int dca_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *plaintext, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_decrypt_final: started"); /* check mechanism */ switch (DCA_MECH_FROM_CTX(ctx)) { case DES_CBC_MECH_INFO_TYPE: error = dca_3desfinal(ctx, plaintext, DR_DECRYPT); break; case DES3_CBC_MECH_INFO_TYPE: error = dca_3desfinal(ctx, plaintext, DR_DECRYPT | DR_TRIPLE); break; default: /* Should never reach here */ cmn_err(CE_WARN, "dca_decrypt_final: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_decrypt_final: done, err = 0x%x", error); return (error); } /* ARGSUSED */ static int dca_decrypt_atomic(crypto_provider_handle_t provider, crypto_session_id_t session_id, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext, crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc = (dca_t *)provider; DBG(softc, DENTRY, "dca_decrypt_atomic: started"); if (ctx_template != NULL) return (CRYPTO_ARGUMENTS_BAD); /* handle inplace ops */ if (!plaintext) { plaintext = ciphertext; } /* check mechanism */ switch (mechanism->cm_type) { case DES_CBC_MECH_INFO_TYPE: error = dca_3desatomic(provider, session_id, mechanism, key, ciphertext, plaintext, KM_SLEEP, req, DR_DECRYPT | DR_ATOMIC); break; case DES3_CBC_MECH_INFO_TYPE: error = dca_3desatomic(provider, session_id, mechanism, key, ciphertext, plaintext, KM_SLEEP, req, DR_DECRYPT | DR_TRIPLE | DR_ATOMIC); break; case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsaatomic(provider, session_id, mechanism, key, ciphertext, plaintext, KM_SLEEP, req, DCA_RSA_DEC); break; default: cmn_err(CE_WARN, "dca_decrypt_atomic: unexpected mech type " "0x%llx\n", (unsigned long long)mechanism->cm_type); error = CRYPTO_MECHANISM_INVALID; } if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) { plaintext->cd_length = 0; } DBG(softc, DENTRY, "dca_decrypt_atomic: done, err = 0x%x", error); return (error); } /* * Sign entry points. */ /* ARGSUSED */ static int dca_sign_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_sign_init: started\n"); if (ctx_template != NULL) return (CRYPTO_ARGUMENTS_BAD); /* check mechanism */ switch (mechanism->cm_type) { case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); break; case DSA_MECH_INFO_TYPE: error = dca_dsainit(ctx, mechanism, key, KM_SLEEP, DCA_DSA_SIGN); break; default: cmn_err(CE_WARN, "dca_sign_init: unexpected mech type " "0x%llx\n", (unsigned long long)mechanism->cm_type); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_sign_init: done, err = 0x%x", error); if (error == CRYPTO_SUCCESS) dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, &softc->dca_ctx_list_lock); return (error); } static int dca_sign(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_sign: started\n"); /* check mechanism */ switch (DCA_MECH_FROM_CTX(ctx)) { case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGN); break; case DSA_MECH_INFO_TYPE: error = dca_dsa_sign(ctx, data, signature, req); break; default: cmn_err(CE_WARN, "dca_sign: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_sign: done, err = 0x%x", error); return (error); } /* ARGSUSED */ static int dca_sign_update(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req) { int error = CRYPTO_MECHANISM_INVALID; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_sign_update: started\n"); cmn_err(CE_WARN, "dca_sign_update: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); DBG(softc, DENTRY, "dca_sign_update: done, err = 0x%x", error); return (error); } /* ARGSUSED */ static int dca_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature, crypto_req_handle_t req) { int error = CRYPTO_MECHANISM_INVALID; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_sign_final: started\n"); cmn_err(CE_WARN, "dca_sign_final: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); DBG(softc, DENTRY, "dca_sign_final: done, err = 0x%x", error); return (error); } static int dca_sign_atomic(crypto_provider_handle_t provider, crypto_session_id_t session_id, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature, crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc = (dca_t *)provider; DBG(softc, DENTRY, "dca_sign_atomic: started\n"); if (ctx_template != NULL) return (CRYPTO_ARGUMENTS_BAD); /* check mechanism */ switch (mechanism->cm_type) { case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsaatomic(provider, session_id, mechanism, key, data, signature, KM_SLEEP, req, DCA_RSA_SIGN); break; case DSA_MECH_INFO_TYPE: error = dca_dsaatomic(provider, session_id, mechanism, key, data, signature, KM_SLEEP, req, DCA_DSA_SIGN); break; default: cmn_err(CE_WARN, "dca_sign_atomic: unexpected mech type " "0x%llx\n", (unsigned long long)mechanism->cm_type); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_sign_atomic: done, err = 0x%x", error); return (error); } /* ARGSUSED */ static int dca_sign_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_sign_recover_init: started\n"); if (ctx_template != NULL) return (CRYPTO_ARGUMENTS_BAD); /* check mechanism */ switch (mechanism->cm_type) { case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); break; default: cmn_err(CE_WARN, "dca_sign_recover_init: unexpected mech type " "0x%llx\n", (unsigned long long)mechanism->cm_type); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_sign_recover_init: done, err = 0x%x", error); if (error == CRYPTO_SUCCESS) dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, &softc->dca_ctx_list_lock); return (error); } static int dca_sign_recover(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_sign_recover: started\n"); /* check mechanism */ switch (DCA_MECH_FROM_CTX(ctx)) { case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGNR); break; default: cmn_err(CE_WARN, "dca_sign_recover: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_sign_recover: done, err = 0x%x", error); return (error); } static int dca_sign_recover_atomic(crypto_provider_handle_t provider, crypto_session_id_t session_id, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature, crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc = (dca_t *)provider; DBG(softc, DENTRY, "dca_sign_recover_atomic: started\n"); if (ctx_template != NULL) return (CRYPTO_ARGUMENTS_BAD); /* check mechanism */ switch (mechanism->cm_type) { case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsaatomic(provider, session_id, mechanism, key, data, signature, KM_SLEEP, req, DCA_RSA_SIGNR); break; default: cmn_err(CE_WARN, "dca_sign_recover_atomic: unexpected mech type" " 0x%llx\n", (unsigned long long)mechanism->cm_type); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_sign_recover_atomic: done, err = 0x%x", error); return (error); } /* * Verify entry points. */ /* ARGSUSED */ static int dca_verify_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_verify_init: started\n"); if (ctx_template != NULL) return (CRYPTO_ARGUMENTS_BAD); /* check mechanism */ switch (mechanism->cm_type) { case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); break; case DSA_MECH_INFO_TYPE: error = dca_dsainit(ctx, mechanism, key, KM_SLEEP, DCA_DSA_VRFY); break; default: cmn_err(CE_WARN, "dca_verify_init: unexpected mech type " "0x%llx\n", (unsigned long long)mechanism->cm_type); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_verify_init: done, err = 0x%x", error); if (error == CRYPTO_SUCCESS) dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, &softc->dca_ctx_list_lock); return (error); } static int dca_verify(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_verify: started\n"); /* check mechanism */ switch (DCA_MECH_FROM_CTX(ctx)) { case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFY); break; case DSA_MECH_INFO_TYPE: error = dca_dsa_verify(ctx, data, signature, req); break; default: cmn_err(CE_WARN, "dca_verify: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_verify: done, err = 0x%x", error); return (error); } /* ARGSUSED */ static int dca_verify_update(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req) { int error = CRYPTO_MECHANISM_INVALID; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_verify_update: started\n"); cmn_err(CE_WARN, "dca_verify_update: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); DBG(softc, DENTRY, "dca_verify_update: done, err = 0x%x", error); return (error); } /* ARGSUSED */ static int dca_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature, crypto_req_handle_t req) { int error = CRYPTO_MECHANISM_INVALID; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_verify_final: started\n"); cmn_err(CE_WARN, "dca_verify_final: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); DBG(softc, DENTRY, "dca_verify_final: done, err = 0x%x", error); return (error); } static int dca_verify_atomic(crypto_provider_handle_t provider, crypto_session_id_t session_id, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature, crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc = (dca_t *)provider; DBG(softc, DENTRY, "dca_verify_atomic: started\n"); if (ctx_template != NULL) return (CRYPTO_ARGUMENTS_BAD); /* check mechanism */ switch (mechanism->cm_type) { case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsaatomic(provider, session_id, mechanism, key, signature, data, KM_SLEEP, req, DCA_RSA_VRFY); break; case DSA_MECH_INFO_TYPE: error = dca_dsaatomic(provider, session_id, mechanism, key, data, signature, KM_SLEEP, req, DCA_DSA_VRFY); break; default: cmn_err(CE_WARN, "dca_verify_atomic: unexpected mech type " "0x%llx\n", (unsigned long long)mechanism->cm_type); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_verify_atomic: done, err = 0x%x", error); return (error); } /* ARGSUSED */ static int dca_verify_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) { int error = CRYPTO_MECHANISM_INVALID; dca_t *softc; softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_verify_recover_init: started\n"); if (ctx_template != NULL) return (CRYPTO_ARGUMENTS_BAD); /* check mechanism */ switch (mechanism->cm_type) { case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsainit(ctx, mechanism, key, KM_SLEEP); break; default: cmn_err(CE_WARN, "dca_verify_recover_init: unexpected mech type" " 0x%llx\n", (unsigned long long)mechanism->cm_type); } DBG(softc, DENTRY, "dca_verify_recover_init: done, err = 0x%x", error); if (error == CRYPTO_SUCCESS) dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private, &softc->dca_ctx_list_lock); return (error); } static int dca_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature, crypto_data_t *data, crypto_req_handle_t req) { int error = CRYPTO_MECHANISM_INVALID; dca_t *softc; if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private) return (CRYPTO_OPERATION_NOT_INITIALIZED); softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_verify_recover: started\n"); /* check mechanism */ switch (DCA_MECH_FROM_CTX(ctx)) { case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFYR); break; default: cmn_err(CE_WARN, "dca_verify_recover: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); } DBG(softc, DENTRY, "dca_verify_recover: done, err = 0x%x", error); return (error); } static int dca_verify_recover_atomic(crypto_provider_handle_t provider, crypto_session_id_t session_id, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature, crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) { int error = CRYPTO_MECHANISM_INVALID; dca_t *softc = (dca_t *)provider; DBG(softc, DENTRY, "dca_verify_recover_atomic: started\n"); if (ctx_template != NULL) return (CRYPTO_ARGUMENTS_BAD); /* check mechanism */ switch (mechanism->cm_type) { case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: error = dca_rsaatomic(provider, session_id, mechanism, key, signature, data, KM_SLEEP, req, DCA_RSA_VRFYR); break; default: cmn_err(CE_WARN, "dca_verify_recover_atomic: unexpected mech " "type 0x%llx\n", (unsigned long long)mechanism->cm_type); error = CRYPTO_MECHANISM_INVALID; } DBG(softc, DENTRY, "dca_verify_recover_atomic: done, err = 0x%x", error); return (error); } /* * Random number entry points. */ /* ARGSUSED */ static int dca_generate_random(crypto_provider_handle_t provider, crypto_session_id_t session_id, uchar_t *buf, size_t len, crypto_req_handle_t req) { int error = CRYPTO_FAILED; dca_t *softc = (dca_t *)provider; DBG(softc, DENTRY, "dca_generate_random: started"); error = dca_rng(softc, buf, len, req); DBG(softc, DENTRY, "dca_generate_random: done, err = 0x%x", error); return (error); } /* * Context management entry points. */ int dca_free_context(crypto_ctx_t *ctx) { int error = CRYPTO_SUCCESS; dca_t *softc; softc = DCA_SOFTC_FROM_CTX(ctx); DBG(softc, DENTRY, "dca_free_context: entered"); if (ctx->cc_provider_private == NULL) return (error); dca_rmlist2(ctx->cc_provider_private, &softc->dca_ctx_list_lock); error = dca_free_context_low(ctx); DBG(softc, DENTRY, "dca_free_context: done, err = 0x%x", error); return (error); } static int dca_free_context_low(crypto_ctx_t *ctx) { int error = CRYPTO_SUCCESS; /* check mechanism */ switch (DCA_MECH_FROM_CTX(ctx)) { case DES_CBC_MECH_INFO_TYPE: case DES3_CBC_MECH_INFO_TYPE: dca_3desctxfree(ctx); break; case RSA_PKCS_MECH_INFO_TYPE: case RSA_X_509_MECH_INFO_TYPE: dca_rsactxfree(ctx); break; case DSA_MECH_INFO_TYPE: dca_dsactxfree(ctx); break; default: /* Should never reach here */ cmn_err(CE_WARN, "dca_free_context_low: unexpected mech type " "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx)); error = CRYPTO_MECHANISM_INVALID; } return (error); } /* Free any unfreed private context. It is called in detach. */ static void dca_free_context_list(dca_t *dca) { dca_listnode_t *node; crypto_ctx_t ctx; (void) memset(&ctx, 0, sizeof (ctx)); ctx.cc_provider = dca; while ((node = dca_delist2(&dca->dca_ctx_list, &dca->dca_ctx_list_lock)) != NULL) { ctx.cc_provider_private = node; (void) dca_free_context_low(&ctx); } } static int ext_info_sym(crypto_provider_handle_t prov, crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq) { return (ext_info_base(prov, ext_info, cfreq, IDENT_SYM)); } static int ext_info_asym(crypto_provider_handle_t prov, crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq) { int rv; rv = ext_info_base(prov, ext_info, cfreq, IDENT_ASYM); /* The asymmetric cipher slot supports random */ ext_info->ei_flags |= CRYPTO_EXTF_RNG; return (rv); } /* ARGSUSED */ static int ext_info_base(crypto_provider_handle_t prov, crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id) { dca_t *dca = (dca_t *)prov; int len; /* Label */ (void) sprintf((char *)ext_info->ei_label, "%s/%d %s", ddi_driver_name(dca->dca_dip), ddi_get_instance(dca->dca_dip), id); len = strlen((char *)ext_info->ei_label); (void) memset(ext_info->ei_label + len, ' ', CRYPTO_EXT_SIZE_LABEL - len); /* Manufacturer ID */ (void) sprintf((char *)ext_info->ei_manufacturerID, "%s", DCA_MANUFACTURER_ID); len = strlen((char *)ext_info->ei_manufacturerID); (void) memset(ext_info->ei_manufacturerID + len, ' ', CRYPTO_EXT_SIZE_MANUF - len); /* Model */ (void) sprintf((char *)ext_info->ei_model, dca->dca_model); DBG(dca, DWARN, "kCF MODEL: %s", (char *)ext_info->ei_model); len = strlen((char *)ext_info->ei_model); (void) memset(ext_info->ei_model + len, ' ', CRYPTO_EXT_SIZE_MODEL - len); /* Serial Number. Blank for Deimos */ (void) memset(ext_info->ei_serial_number, ' ', CRYPTO_EXT_SIZE_SERIAL); ext_info->ei_flags = CRYPTO_EXTF_WRITE_PROTECTED; ext_info->ei_max_session_count = CRYPTO_UNAVAILABLE_INFO; ext_info->ei_max_pin_len = CRYPTO_UNAVAILABLE_INFO; ext_info->ei_min_pin_len = CRYPTO_UNAVAILABLE_INFO; ext_info->ei_total_public_memory = CRYPTO_UNAVAILABLE_INFO; ext_info->ei_free_public_memory = CRYPTO_UNAVAILABLE_INFO; ext_info->ei_total_private_memory = CRYPTO_UNAVAILABLE_INFO; ext_info->ei_free_private_memory = CRYPTO_UNAVAILABLE_INFO; ext_info->ei_hardware_version.cv_major = 0; ext_info->ei_hardware_version.cv_minor = 0; ext_info->ei_firmware_version.cv_major = 0; ext_info->ei_firmware_version.cv_minor = 0; /* Time. No need to be supplied for token without a clock */ ext_info->ei_time[0] = '\000'; return (CRYPTO_SUCCESS); } static void dca_fma_init(dca_t *dca) { ddi_iblock_cookie_t fm_ibc; int fm_capabilities = DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE; /* Read FMA capabilities from dca.conf file (if present) */ dca->fm_capabilities = ddi_getprop(DDI_DEV_T_ANY, dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", fm_capabilities); DBG(dca, DWARN, "dca->fm_capabilities = 0x%x", dca->fm_capabilities); /* Only register with IO Fault Services if we have some capability */ if (dca->fm_capabilities) { dca_regsattr.devacc_attr_access = DDI_FLAGERR_ACC; dca_dmaattr.dma_attr_flags = DDI_DMA_FLAGERR; /* Register capabilities with IO Fault Services */ ddi_fm_init(dca->dca_dip, &dca->fm_capabilities, &fm_ibc); DBG(dca, DWARN, "fm_capable() = 0x%x", ddi_fm_capable(dca->dca_dip)); /* * Initialize pci ereport capabilities if ereport capable */ if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) || DDI_FM_ERRCB_CAP(dca->fm_capabilities)) pci_ereport_setup(dca->dca_dip); /* * Initialize callback mutex and register error callback if * error callback capable. */ if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) { ddi_fm_handler_register(dca->dca_dip, dca_fm_error_cb, (void *)dca); } } else { /* * These fields have to be cleared of FMA if there are no * FMA capabilities at runtime. */ dca_regsattr.devacc_attr_access = DDI_DEFAULT_ACC; dca_dmaattr.dma_attr_flags = 0; } } static void dca_fma_fini(dca_t *dca) { /* Only unregister FMA capabilities if we registered some */ if (dca->fm_capabilities) { /* * Release any resources allocated by pci_ereport_setup() */ if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) || DDI_FM_ERRCB_CAP(dca->fm_capabilities)) { pci_ereport_teardown(dca->dca_dip); } /* * Free callback mutex and un-register error callback if * error callback capable. */ if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) { ddi_fm_handler_unregister(dca->dca_dip); } /* Unregister from IO Fault Services */ ddi_fm_fini(dca->dca_dip); DBG(dca, DWARN, "fm_capable() = 0x%x", ddi_fm_capable(dca->dca_dip)); } } /* * The IO fault service error handling callback function */ /*ARGSUSED*/ static int dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) { dca_t *dca = (dca_t *)impl_data; pci_ereport_post(dip, err, NULL); if (err->fme_status == DDI_FM_FATAL) { dca_failure(dca, DDI_DATAPATH_FAULT, DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR, "fault PCI in FMA callback."); } return (err->fme_status); } static int dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle, dca_fma_eclass_t eclass_index) { ddi_fm_error_t de; int version = 0; ddi_fm_acc_err_get(handle, &de, version); if (de.fme_status != DDI_FM_OK) { dca_failure(dca, DDI_DATAPATH_FAULT, eclass_index, fm_ena_increment(de.fme_ena), CRYPTO_DEVICE_ERROR, ""); return (DDI_FAILURE); } return (DDI_SUCCESS); } int dca_check_dma_handle(dca_t *dca, ddi_dma_handle_t handle, dca_fma_eclass_t eclass_index) { ddi_fm_error_t de; int version = 0; ddi_fm_dma_err_get(handle, &de, version); if (de.fme_status != DDI_FM_OK) { dca_failure(dca, DDI_DATAPATH_FAULT, eclass_index, fm_ena_increment(de.fme_ena), CRYPTO_DEVICE_ERROR, ""); return (DDI_FAILURE); } return (DDI_SUCCESS); } static uint64_t dca_ena(uint64_t ena) { if (ena == 0) ena = fm_ena_generate(0, FM_ENA_FMT1); else ena = fm_ena_increment(ena); return (ena); } static char * dca_fma_eclass_string(char *model, dca_fma_eclass_t index) { if (strstr(model, "500")) return (dca_fma_eclass_sca500[index]); else return (dca_fma_eclass_sca1000[index]); }