/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" /* * PCI nexus driver interface */ #include #include /* nulldev */ #include /* devctl */ #include #include /* ecc_flt for pci_ecc.h */ #include #include #include #include #include #include #include #include #include #include /*LINTLIBRARY*/ /* * function prototype for hotplug routine: */ static void pci_init_hotplug(struct pci *); /* * function prototypes for dev ops routines: */ static int pci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); static int pci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); static int pci_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result); static int pci_ctlops_poke(pci_t *pci_p, peekpoke_ctlops_t *in_args); static int pci_ctlops_peek(pci_t *pci_p, peekpoke_ctlops_t *in_args, void *result); static off_t get_reg_set_size(dev_info_t *child, int rnumber); /* * bus ops and dev ops structures: */ static struct bus_ops pci_bus_ops = { BUSO_REV, pci_map, 0, 0, 0, i_ddi_map_fault, pci_dma_setup, pci_dma_allochdl, pci_dma_freehdl, pci_dma_bindhdl, pci_dma_unbindhdl, pci_dma_sync, pci_dma_win, pci_dma_ctlops, pci_ctlops, ddi_bus_prop_op, ndi_busop_get_eventcookie, /* (*bus_get_eventcookie)(); */ ndi_busop_add_eventcall, /* (*bus_add_eventcall)(); */ ndi_busop_remove_eventcall, /* (*bus_remove_eventcall)(); */ ndi_post_event, /* (*bus_post_event)(); */ NULL, /* (*bus_intr_ctl)(); */ NULL, /* (*bus_config)(); */ NULL, /* (*bus_unconfig)(); */ pci_fm_init_child, /* (*bus_fm_init)(); */ NULL, /* (*bus_fm_fini)(); */ pci_bus_enter, /* (*bus_fm_access_enter)(); */ pci_bus_exit, /* (*bus_fm_access_fini)(); */ NULL, /* (*bus_power)(); */ pci_intr_ops /* (*bus_intr_op)(); */ }; extern struct cb_ops pci_cb_ops; static struct dev_ops pci_ops = { DEVO_REV, 0, pci_info, nulldev, 0, pci_attach, pci_detach, nodev, &pci_cb_ops, &pci_bus_ops, 0 }; /* * module definitions: */ #include extern struct mod_ops mod_driverops; static struct modldrv modldrv = { &mod_driverops, /* Type of module - driver */ "PCI Bus nexus driver %I%", /* Name of module. */ &pci_ops, /* driver ops */ }; static struct modlinkage modlinkage = { MODREV_1, (void *)&modldrv, NULL }; /* * driver global data: */ void *per_pci_state; /* per-pbm soft state pointer */ void *per_pci_common_state; /* per-psycho soft state pointer */ kmutex_t pci_global_mutex; /* attach/detach common struct lock */ errorq_t *pci_ecc_queue = NULL; /* per-system ecc handling queue */ errorq_t *pci_target_queue = NULL; /* per-system target handling queue */ extern void pci_child_cfg_save(dev_info_t *dip); extern void pci_child_cfg_restore(dev_info_t *dip); int _init(void) { int e; /* * Initialize per-pci bus soft state pointer. */ e = ddi_soft_state_init(&per_pci_state, sizeof (pci_t), 1); if (e != 0) return (e); /* * Initialize per-psycho soft state pointer. */ e = ddi_soft_state_init(&per_pci_common_state, sizeof (pci_common_t), 1); if (e != 0) { ddi_soft_state_fini(&per_pci_state); return (e); } /* * Initialize global mutexes. */ mutex_init(&pci_global_mutex, NULL, MUTEX_DRIVER, NULL); pci_reloc_init(); /* * Create the performance kstats. */ pci_kstat_init(); /* * Install the module. */ e = mod_install(&modlinkage); if (e != 0) { ddi_soft_state_fini(&per_pci_state); ddi_soft_state_fini(&per_pci_common_state); mutex_destroy(&pci_global_mutex); } return (e); } int _fini(void) { int e; /* * Remove the module. */ e = mod_remove(&modlinkage); if (e != 0) return (e); /* * Destroy pci_ecc_queue, and set it to NULL. */ if (pci_ecc_queue) errorq_destroy(pci_ecc_queue); pci_ecc_queue = NULL; /* * Destroy pci_target_queue, and set it to NULL. */ if (pci_target_queue) errorq_destroy(pci_target_queue); pci_target_queue = NULL; /* * Destroy the performance kstats. */ pci_kstat_fini(); /* * Free the per-pci and per-psycho soft state info and destroy * mutex for per-psycho soft state. */ ddi_soft_state_fini(&per_pci_state); ddi_soft_state_fini(&per_pci_common_state); mutex_destroy(&pci_global_mutex); pci_reloc_fini(); return (e); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } /*ARGSUSED*/ static int pci_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) { int instance = PCIHP_AP_MINOR_NUM_TO_INSTANCE(getminor((dev_t)arg)); pci_t *pci_p = get_pci_soft_state(instance); /* allow hotplug to deal with ones it manages */ if (pci_p && (pci_p->hotplug_capable == B_TRUE)) return (pcihp_info(dip, infocmd, arg, result)); /* non-hotplug or not attached */ switch (infocmd) { case DDI_INFO_DEVT2INSTANCE: *result = (void *)instance; return (DDI_SUCCESS); case DDI_INFO_DEVT2DEVINFO: if (pci_p == NULL) return (DDI_FAILURE); *result = (void *)pci_p->pci_dip; return (DDI_SUCCESS); default: return (DDI_FAILURE); } } /* device driver entry points */ /* * attach entry point: */ static int pci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { pci_t *pci_p; /* per bus state pointer */ int instance = ddi_get_instance(dip); switch (cmd) { case DDI_ATTACH: DEBUG0(DBG_ATTACH, dip, "DDI_ATTACH\n"); /* * Allocate and get the per-pci soft state structure. */ if (alloc_pci_soft_state(instance) != DDI_SUCCESS) { cmn_err(CE_WARN, "%s%d: can't allocate pci state", ddi_driver_name(dip), instance); goto err_bad_pci_softstate; } pci_p = get_pci_soft_state(instance); pci_p->pci_dip = dip; mutex_init(&pci_p->pci_mutex, NULL, MUTEX_DRIVER, NULL); pci_p->pci_soft_state = PCI_SOFT_STATE_CLOSED; pci_p->pci_open_count = 0; /* * Get key properties of the pci bridge node and * determine it's type (psycho, schizo, etc ...). */ if (get_pci_properties(pci_p, dip) == DDI_FAILURE) goto err_bad_pci_prop; /* * Map in the registers. */ if (map_pci_registers(pci_p, dip) == DDI_FAILURE) goto err_bad_reg_prop; if (pci_obj_setup(pci_p) != DDI_SUCCESS) goto err_bad_objs; /* * If this PCI leaf has hotplug and this platform * loads hotplug modules then initialize the * hotplug framework. */ pci_init_hotplug(pci_p); /* * Create the "devctl" node for hotplug support. * For non-hotplug bus, we still need ":devctl" to * support DEVCTL_DEVICE_* and DEVCTL_BUS_* ioctls. */ if (pci_p->hotplug_capable == B_FALSE) { if (ddi_create_minor_node(dip, "devctl", S_IFCHR, PCIHP_AP_MINOR_NUM(instance, PCIHP_DEVCTL_MINOR), DDI_NT_NEXUS, 0) != DDI_SUCCESS) goto err_bad_devctl_node; } /* * Due to unresolved hardware issues, disable PCIPM until * the problem is fully understood. * * pci_pwr_setup(pci_p, dip); */ ddi_report_dev(dip); pci_p->pci_state = PCI_ATTACHED; DEBUG0(DBG_ATTACH, dip, "attach success\n"); break; err_bad_objs: ddi_remove_minor_node(dip, "devctl"); err_bad_devctl_node: unmap_pci_registers(pci_p); err_bad_reg_prop: free_pci_properties(pci_p); err_bad_pci_prop: mutex_destroy(&pci_p->pci_mutex); free_pci_soft_state(instance); err_bad_pci_softstate: return (DDI_FAILURE); case DDI_RESUME: DEBUG0(DBG_ATTACH, dip, "DDI_RESUME\n"); /* * Make sure the Psycho control registers and IOMMU * are configured properly. */ pci_p = get_pci_soft_state(instance); mutex_enter(&pci_p->pci_mutex); /* * Make sure this instance has been suspended. */ if (pci_p->pci_state != PCI_SUSPENDED) { DEBUG0(DBG_ATTACH, dip, "instance NOT suspended\n"); mutex_exit(&pci_p->pci_mutex); return (DDI_FAILURE); } pci_obj_resume(pci_p); pci_p->pci_state = PCI_ATTACHED; pci_child_cfg_restore(dip); mutex_exit(&pci_p->pci_mutex); break; default: DEBUG0(DBG_ATTACH, dip, "unsupported attach op\n"); return (DDI_FAILURE); } return (DDI_SUCCESS); } /* * detach entry point: */ static int pci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { int instance = ddi_get_instance(dip); pci_t *pci_p = get_pci_soft_state(instance); /* * Make sure we are currently attached */ if (pci_p->pci_state != PCI_ATTACHED) { DEBUG0(DBG_ATTACH, dip, "failed - instance not attached\n"); return (DDI_FAILURE); } mutex_enter(&pci_p->pci_mutex); switch (cmd) { case DDI_DETACH: DEBUG0(DBG_DETACH, dip, "DDI_DETACH\n"); if (pci_p->hotplug_capable == B_TRUE) if (pcihp_uninit(dip) == DDI_FAILURE) { mutex_exit(&pci_p->pci_mutex); return (DDI_FAILURE); } pci_obj_destroy(pci_p); /* * Free the pci soft state structure and the rest of the * resources it's using. */ free_pci_properties(pci_p); unmap_pci_registers(pci_p); mutex_exit(&pci_p->pci_mutex); mutex_destroy(&pci_p->pci_mutex); free_pci_soft_state(instance); /* Free the interrupt-priorities prop if we created it. */ { int len; if (ddi_getproplen(DDI_DEV_T_ANY, dip, DDI_PROP_NOTPROM | DDI_PROP_DONTPASS, "interrupt-priorities", &len) == DDI_PROP_SUCCESS) (void) ddi_prop_remove(DDI_DEV_T_NONE, dip, "interrupt-priorities"); } return (DDI_SUCCESS); case DDI_SUSPEND: pci_child_cfg_save(dip); pci_obj_suspend(pci_p); pci_p->pci_state = PCI_SUSPENDED; mutex_exit(&pci_p->pci_mutex); return (DDI_SUCCESS); default: DEBUG0(DBG_DETACH, dip, "unsupported detach op\n"); mutex_exit(&pci_p->pci_mutex); return (DDI_FAILURE); } } /* bus driver entry points */ /* * bus map entry point: * * if map request is for an rnumber * get the corresponding regspec from device node * build a new regspec in our parent's format * build a new map_req with the new regspec * call up the tree to complete the mapping */ int pci_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t off, off_t len, caddr_t *addrp) { pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip)); struct regspec p_regspec; ddi_map_req_t p_mapreq; int reglen, rval, r_no; pci_regspec_t reloc_reg, *rp = &reloc_reg; DEBUG2(DBG_MAP, dip, "rdip=%s%d:", ddi_driver_name(rdip), ddi_get_instance(rdip)); if (mp->map_flags & DDI_MF_USER_MAPPING) return (DDI_ME_UNIMPLEMENTED); switch (mp->map_type) { case DDI_MT_REGSPEC: reloc_reg = *(pci_regspec_t *)mp->map_obj.rp; /* dup whole */ break; case DDI_MT_RNUMBER: r_no = mp->map_obj.rnumber; DEBUG1(DBG_MAP | DBG_CONT, dip, " r#=%x", r_no); if (ddi_getlongprop(DDI_DEV_T_NONE, rdip, DDI_PROP_DONTPASS, "reg", (caddr_t)&rp, ®len) != DDI_SUCCESS) return (DDI_ME_RNUMBER_RANGE); if (r_no < 0 || r_no >= reglen / sizeof (pci_regspec_t)) { kmem_free(rp, reglen); return (DDI_ME_RNUMBER_RANGE); } rp += r_no; break; default: return (DDI_ME_INVAL); } DEBUG0(DBG_MAP | DBG_CONT, dip, "\n"); /* use "assigned-addresses" to relocate regspec within pci space */ if (rval = pci_reloc_reg(dip, rdip, pci_p, rp)) goto done; if (len) /* adjust regspec according to mapping request */ rp->pci_size_low = len; rp->pci_phys_low += off; /* use "ranges" to translate relocated pci regspec into parent space */ if (rval = pci_xlate_reg(pci_p, rp, &p_regspec)) goto done; p_mapreq = *mp; /* dup the whole structure */ p_mapreq.map_type = DDI_MT_REGSPEC; p_mapreq.map_obj.rp = &p_regspec; rval = ddi_map(dip, &p_mapreq, 0, 0, addrp); if (rval == DDI_SUCCESS) { /* * Set-up access functions for FM access error capable drivers. * The axq workaround prevents fault management support */ if (DDI_FM_ACC_ERR_CAP(pci_p->pci_fm_cap) && DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) && mp->map_handlep->ah_acc.devacc_attr_access != DDI_DEFAULT_ACC) pci_fm_acc_setup(mp, rdip); pci_axq_setup(mp, pci_p->pci_pbm_p); } done: if (mp->map_type == DDI_MT_RNUMBER) kmem_free(rp - r_no, reglen); return (rval); } /* * bus dma map entry point * return value: * DDI_DMA_PARTIAL_MAP 1 * DDI_DMA_MAPOK 0 * DDI_DMA_MAPPED 0 * DDI_DMA_NORESOURCES -1 * DDI_DMA_NOMAPPING -2 * DDI_DMA_TOOBIG -3 */ int pci_dma_setup(dev_info_t *dip, dev_info_t *rdip, ddi_dma_req_t *dmareq, ddi_dma_handle_t *handlep) { pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip)); iommu_t *iommu_p = pci_p->pci_iommu_p; ddi_dma_impl_t *mp; int ret; DEBUG3(DBG_DMA_MAP, dip, "mapping - rdip=%s%d type=%s\n", ddi_driver_name(rdip), ddi_get_instance(rdip), handlep ? "alloc" : "advisory"); if (!(mp = pci_dma_lmts2hdl(dip, rdip, iommu_p, dmareq))) return (DDI_DMA_NORESOURCES); if (mp == (ddi_dma_impl_t *)DDI_DMA_NOMAPPING) return (DDI_DMA_NOMAPPING); if (ret = pci_dma_type(pci_p, dmareq, mp)) goto freehandle; if (ret = pci_dma_pfn(pci_p, dmareq, mp)) goto freehandle; switch (PCI_DMA_TYPE(mp)) { case DMAI_FLAGS_DVMA: /* LINTED E_EQUALITY_NOT_ASSIGNMENT */ if ((ret = pci_dvma_win(pci_p, dmareq, mp)) || !handlep) goto freehandle; if (!PCI_DMA_CANCACHE(mp)) { /* try fast track */ if (PCI_DMA_CANFAST(mp)) { if (!pci_dvma_map_fast(iommu_p, mp)) break; /* LINTED E_NOP_ELSE_STMT */ } else { PCI_DVMA_FASTTRAK_PROF(mp); } } if (ret = pci_dvma_map(mp, dmareq, iommu_p)) goto freehandle; break; case DMAI_FLAGS_PEER_TO_PEER: /* LINTED E_EQUALITY_NOT_ASSIGNMENT */ if ((ret = pci_dma_physwin(pci_p, dmareq, mp)) || !handlep) goto freehandle; break; case DMAI_FLAGS_BYPASS: default: panic("%s%d: pci_dma_setup: bad dma type 0x%x", ddi_driver_name(rdip), ddi_get_instance(rdip), PCI_DMA_TYPE(mp)); /*NOTREACHED*/ } *handlep = (ddi_dma_handle_t)mp; mp->dmai_flags |= (DMAI_FLAGS_INUSE | DMAI_FLAGS_MAPPED); dump_dma_handle(DBG_DMA_MAP, dip, mp); return ((mp->dmai_nwin == 1) ? DDI_DMA_MAPPED : DDI_DMA_PARTIAL_MAP); freehandle: if (ret == DDI_DMA_NORESOURCES) pci_dma_freemp(mp); /* don't run_callback() */ else (void) pci_dma_freehdl(dip, rdip, (ddi_dma_handle_t)mp); return (ret); } /* * bus dma alloc handle entry point: */ int pci_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attrp, int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) { pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip)); ddi_dma_impl_t *mp; int rval; DEBUG2(DBG_DMA_ALLOCH, dip, "rdip=%s%d\n", ddi_driver_name(rdip), ddi_get_instance(rdip)); if (attrp->dma_attr_version != DMA_ATTR_V0) return (DDI_DMA_BADATTR); if (!(mp = pci_dma_allocmp(dip, rdip, waitfp, arg))) return (DDI_DMA_NORESOURCES); /* * Save requestor's information */ mp->dmai_attr = *attrp; /* whole object - augmented later */ *DEV_ATTR(mp) = *attrp; /* whole object - device orig attr */ DEBUG1(DBG_DMA_ALLOCH, dip, "mp=%p\n", mp); /* check and convert dma attributes to handle parameters */ if (rval = pci_dma_attr2hdl(pci_p, mp)) { pci_dma_freehdl(dip, rdip, (ddi_dma_handle_t)mp); *handlep = NULL; return (rval); } *handlep = (ddi_dma_handle_t)mp; return (DDI_SUCCESS); } /* * bus dma free handle entry point: */ /*ARGSUSED*/ int pci_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) { DEBUG3(DBG_DMA_FREEH, dip, "rdip=%s%d mp=%p\n", ddi_driver_name(rdip), ddi_get_instance(rdip), handle); pci_dma_freemp((ddi_dma_impl_t *)handle); if (pci_kmem_clid) { DEBUG0(DBG_DMA_FREEH, dip, "run handle callback\n"); ddi_run_callback(&pci_kmem_clid); } return (DDI_SUCCESS); } /* * bus dma bind handle entry point: */ int pci_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, ddi_dma_req_t *dmareq, ddi_dma_cookie_t *cookiep, uint_t *ccountp) { pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip)); iommu_t *iommu_p = pci_p->pci_iommu_p; ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; int ret; DEBUG4(DBG_DMA_BINDH, dip, "rdip=%s%d mp=%p dmareq=%p\n", ddi_driver_name(rdip), ddi_get_instance(rdip), mp, dmareq); if (mp->dmai_flags & DMAI_FLAGS_INUSE) return (DDI_DMA_INUSE); ASSERT((mp->dmai_flags & ~DMAI_FLAGS_PRESERVE) == 0); mp->dmai_flags |= DMAI_FLAGS_INUSE; if (ret = pci_dma_type(pci_p, dmareq, mp)) goto err; if (ret = pci_dma_pfn(pci_p, dmareq, mp)) goto err; switch (PCI_DMA_TYPE(mp)) { case DMAI_FLAGS_DVMA: if (ret = pci_dvma_win(pci_p, dmareq, mp)) goto map_err; if (!PCI_DMA_CANCACHE(mp)) { /* try fast track */ if (PCI_DMA_CANFAST(mp)) { if (!pci_dvma_map_fast(iommu_p, mp)) goto mapped; /*LINTED E_NOP_ELSE_STMT*/ } else { PCI_DVMA_FASTTRAK_PROF(mp); } } if (ret = pci_dvma_map(mp, dmareq, iommu_p)) goto map_err; mapped: *ccountp = 1; MAKE_DMA_COOKIE(cookiep, mp->dmai_mapping, mp->dmai_size); break; case DMAI_FLAGS_BYPASS: case DMAI_FLAGS_PEER_TO_PEER: if (ret = pci_dma_physwin(pci_p, dmareq, mp)) goto map_err; *ccountp = WINLST(mp)->win_ncookies; *cookiep = *(ddi_dma_cookie_t *)(WINLST(mp) + 1); /* wholeobj */ break; default: panic("%s%d: pci_dma_bindhdl(%p): bad dma type", ddi_driver_name(rdip), ddi_get_instance(rdip), mp); /*NOTREACHED*/ } DEBUG2(DBG_DMA_BINDH, dip, "cookie %x+%x\n", cookiep->dmac_address, cookiep->dmac_size); dump_dma_handle(DBG_DMA_MAP, dip, mp); if (mp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) (void) ndi_fmc_insert(rdip, DMA_HANDLE, mp, NULL); mp->dmai_flags |= DMAI_FLAGS_MAPPED; return (mp->dmai_nwin == 1 ? DDI_DMA_MAPPED : DDI_DMA_PARTIAL_MAP); map_err: pci_dvma_unregister_callbacks(pci_p, mp); pci_dma_freepfn(mp); err: mp->dmai_flags &= DMAI_FLAGS_PRESERVE; return (ret); } /* * bus dma unbind handle entry point: */ /*ARGSUSED*/ int pci_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) { ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip)); iommu_t *iommu_p = pci_p->pci_iommu_p; DEBUG3(DBG_DMA_UNBINDH, dip, "rdip=%s%d, mp=%p\n", ddi_driver_name(rdip), ddi_get_instance(rdip), handle); if ((mp->dmai_flags & DMAI_FLAGS_INUSE) == 0) { DEBUG0(DBG_DMA_UNBINDH, dip, "handle not in use\n"); return (DDI_FAILURE); } mp->dmai_flags &= ~DMAI_FLAGS_MAPPED; switch (PCI_DMA_TYPE(mp)) { case DMAI_FLAGS_DVMA: pci_dvma_unregister_callbacks(pci_p, mp); pci_dma_sync_unmap(dip, rdip, mp); pci_dvma_unmap(iommu_p, mp); pci_dma_freepfn(mp); break; case DMAI_FLAGS_BYPASS: case DMAI_FLAGS_PEER_TO_PEER: pci_dma_freewin(mp); break; default: panic("%s%d: pci_dma_unbindhdl:bad dma type %p", ddi_driver_name(rdip), ddi_get_instance(rdip), mp); /*NOTREACHED*/ } if (iommu_p->iommu_dvma_clid != 0) { DEBUG0(DBG_DMA_UNBINDH, dip, "run dvma callback\n"); ddi_run_callback(&iommu_p->iommu_dvma_clid); } if (pci_kmem_clid) { DEBUG0(DBG_DMA_UNBINDH, dip, "run handle callback\n"); ddi_run_callback(&pci_kmem_clid); } mp->dmai_flags &= DMAI_FLAGS_PRESERVE; SYNC_BUF_PA(mp) = 0; if (mp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) { if (DEVI(rdip)->devi_fmhdl != NULL && DDI_FM_DMA_ERR_CAP(DEVI(rdip)->devi_fmhdl->fh_cap)) { (void) ndi_fmc_remove(rdip, DMA_HANDLE, mp); } } return (DDI_SUCCESS); } /* * bus dma win entry point: */ int pci_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, uint_t *ccountp) { ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; DEBUG2(DBG_DMA_WIN, dip, "rdip=%s%d\n", ddi_driver_name(rdip), ddi_get_instance(rdip)); dump_dma_handle(DBG_DMA_WIN, dip, mp); if (win >= mp->dmai_nwin) { DEBUG1(DBG_DMA_WIN, dip, "%x out of range\n", win); return (DDI_FAILURE); } switch (PCI_DMA_TYPE(mp)) { case DMAI_FLAGS_DVMA: if (win != PCI_DMA_CURWIN(mp)) { pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip)); pci_dma_sync_unmap(dip, rdip, mp); /* map_window sets dmai_mapping/size/offset */ iommu_map_window(pci_p->pci_iommu_p, mp, win); } if (cookiep) MAKE_DMA_COOKIE(cookiep, mp->dmai_mapping, mp->dmai_size); if (ccountp) *ccountp = 1; break; case DMAI_FLAGS_PEER_TO_PEER: case DMAI_FLAGS_BYPASS: { int i; ddi_dma_cookie_t *ck_p; pci_dma_win_t *win_p = mp->dmai_winlst; for (i = 0; i < win; win_p = win_p->win_next, i++); ck_p = (ddi_dma_cookie_t *)(win_p + 1); *cookiep = *ck_p; mp->dmai_offset = win_p->win_offset; mp->dmai_size = win_p->win_size; mp->dmai_mapping = ck_p->dmac_laddress; mp->dmai_cookie = ck_p + 1; win_p->win_curseg = 0; if (ccountp) *ccountp = win_p->win_ncookies; } break; default: cmn_err(CE_WARN, "%s%d: pci_dma_win:bad dma type 0x%x", ddi_driver_name(rdip), ddi_get_instance(rdip), PCI_DMA_TYPE(mp)); return (DDI_FAILURE); } if (cookiep) DEBUG2(DBG_DMA_WIN, dip, "cookie - dmac_address=%x dmac_size=%x\n", cookiep->dmac_address, cookiep->dmac_size); if (offp) *offp = (off_t)mp->dmai_offset; if (lenp) *lenp = mp->dmai_size; return (DDI_SUCCESS); } #ifdef DEBUG static char *pci_dmactl_str[] = { "DDI_DMA_FREE", "DDI_DMA_SYNC", "DDI_DMA_HTOC", "DDI_DMA_KVADDR", "DDI_DMA_MOVWIN", "DDI_DMA_REPWIN", "DDI_DMA_GETERR", "DDI_DMA_COFF", "DDI_DMA_NEXTWIN", "DDI_DMA_NEXTSEG", "DDI_DMA_SEGTOC", "DDI_DMA_RESERVE", "DDI_DMA_RELEASE", "DDI_DMA_RESETH", "DDI_DMA_CKSYNC", "DDI_DMA_IOPB_ALLOC", "DDI_DMA_IOPB_FREE", "DDI_DMA_SMEM_ALLOC", "DDI_DMA_SMEM_FREE", "DDI_DMA_SET_SBUS64", "DDI_DMA_REMAP" }; #endif /* * bus dma control entry point: */ int pci_dma_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags) { ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; DEBUG3(DBG_DMA_CTL, dip, "%s: rdip=%s%d\n", pci_dmactl_str[cmd], ddi_driver_name(rdip), ddi_get_instance(rdip)); switch (cmd) { case DDI_DMA_FREE: (void) pci_dma_unbindhdl(dip, rdip, handle); (void) pci_dma_freehdl(dip, rdip, handle); return (DDI_SUCCESS); case DDI_DMA_RESERVE: { pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip)); return (pci_fdvma_reserve(dip, rdip, pci_p, (ddi_dma_req_t *)offp, (ddi_dma_handle_t *)objp)); } case DDI_DMA_RELEASE: { pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip)); return (pci_fdvma_release(dip, pci_p, mp)); } default: break; } switch (PCI_DMA_TYPE(mp)) { case DMAI_FLAGS_DVMA: return (pci_dvma_ctl(dip, rdip, mp, cmd, offp, lenp, objp, cache_flags)); case DMAI_FLAGS_PEER_TO_PEER: case DMAI_FLAGS_BYPASS: return (pci_dma_ctl(dip, rdip, mp, cmd, offp, lenp, objp, cache_flags)); default: panic("%s%d: pci_dma_ctlops(%x):bad dma type %x", ddi_driver_name(rdip), ddi_get_instance(rdip), cmd, mp->dmai_flags); /*NOTREACHED*/ } } #ifdef DEBUG int pci_peekfault_cnt = 0; int pci_pokefault_cnt = 0; #endif /* DEBUG */ static int pci_do_poke(pci_t *pci_p, peekpoke_ctlops_t *in_args) { pbm_t *pbm_p = pci_p->pci_pbm_p; int err = DDI_SUCCESS; on_trap_data_t otd; mutex_enter(&pbm_p->pbm_pokefault_mutex); pbm_p->pbm_ontrap_data = &otd; /* Set up protected environment. */ if (!on_trap(&otd, OT_DATA_ACCESS)) { uintptr_t tramp = otd.ot_trampoline; otd.ot_trampoline = (uintptr_t)&poke_fault; err = do_poke(in_args->size, (void *)in_args->dev_addr, (void *)in_args->host_addr); otd.ot_trampoline = tramp; } else err = DDI_FAILURE; /* * Read the async fault register for the PBM to see it sees * a master-abort. */ pbm_clear_error(pbm_p); if (otd.ot_trap & OT_DATA_ACCESS) err = DDI_FAILURE; /* Take down protected environment. */ no_trap(); pbm_p->pbm_ontrap_data = NULL; mutex_exit(&pbm_p->pbm_pokefault_mutex); #ifdef DEBUG if (err == DDI_FAILURE) pci_pokefault_cnt++; #endif return (err); } static int pci_do_caut_put(pci_t *pci_p, peekpoke_ctlops_t *cautacc_ctlops_arg) { size_t size = cautacc_ctlops_arg->size; uintptr_t dev_addr = cautacc_ctlops_arg->dev_addr; uintptr_t host_addr = cautacc_ctlops_arg->host_addr; ddi_acc_impl_t *hp = (ddi_acc_impl_t *)cautacc_ctlops_arg->handle; size_t repcount = cautacc_ctlops_arg->repcount; uint_t flags = cautacc_ctlops_arg->flags; pbm_t *pbm_p = pci_p->pci_pbm_p; int err = DDI_SUCCESS; /* Use ontrap data in handle set up by FMA */ pbm_p->pbm_ontrap_data = (on_trap_data_t *)hp->ahi_err->err_ontrap; hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED; /* * Note that i_ndi_busop_access_enter ends up grabbing the pokefault * mutex. */ i_ndi_busop_access_enter(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp); if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) { for (; repcount; repcount--) { switch (size) { case sizeof (uint8_t): i_ddi_put8(hp, (uint8_t *)dev_addr, *(uint8_t *)host_addr); break; case sizeof (uint16_t): i_ddi_put16(hp, (uint16_t *)dev_addr, *(uint16_t *)host_addr); break; case sizeof (uint32_t): i_ddi_put32(hp, (uint32_t *)dev_addr, *(uint32_t *)host_addr); break; case sizeof (uint64_t): i_ddi_put64(hp, (uint64_t *)dev_addr, *(uint64_t *)host_addr); break; } host_addr += size; if (flags == DDI_DEV_AUTOINCR) dev_addr += size; /* * Read the async fault register for the PBM to see if * it sees a master-abort. */ pbm_clear_error(pbm_p); if (pbm_p->pbm_ontrap_data->ot_trap & OT_DATA_ACCESS) { err = DDI_FAILURE; #ifdef DEBUG pci_pokefault_cnt++; #endif break; } } } i_ddi_notrap((ddi_acc_handle_t)hp); pbm_p->pbm_ontrap_data = NULL; i_ndi_busop_access_exit(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp); hp->ahi_err->err_expected = DDI_FM_ERR_UNEXPECTED; return (err); } static int pci_ctlops_poke(pci_t *pci_p, peekpoke_ctlops_t *in_args) { return (in_args->handle ? pci_do_caut_put(pci_p, in_args) : pci_do_poke(pci_p, in_args)); } static int pci_do_peek(pci_t *pci_p, peekpoke_ctlops_t *in_args) { int err = DDI_SUCCESS; on_trap_data_t otd; if (!on_trap(&otd, OT_DATA_ACCESS)) { uintptr_t tramp = otd.ot_trampoline; otd.ot_trampoline = (uintptr_t)&peek_fault; err = do_peek(in_args->size, (void *)in_args->dev_addr, (void *)in_args->host_addr); otd.ot_trampoline = tramp; } else err = DDI_FAILURE; no_trap(); #ifdef DEBUG if (err == DDI_FAILURE) pci_peekfault_cnt++; #endif return (err); } static int pci_do_caut_get(pci_t *pci_p, peekpoke_ctlops_t *cautacc_ctlops_arg) { size_t size = cautacc_ctlops_arg->size; uintptr_t dev_addr = cautacc_ctlops_arg->dev_addr; uintptr_t host_addr = cautacc_ctlops_arg->host_addr; ddi_acc_impl_t *hp = (ddi_acc_impl_t *)cautacc_ctlops_arg->handle; size_t repcount = cautacc_ctlops_arg->repcount; uint_t flags = cautacc_ctlops_arg->flags; pbm_t *pbm_p = pci_p->pci_pbm_p; int err = DDI_SUCCESS; hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED; i_ndi_busop_access_enter(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp); /* Can this code be optimized? */ if (repcount == 1) { if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) { i_ddi_caut_get(size, (void *)dev_addr, (void *)host_addr); } else { int i; uint8_t *ff_addr = (uint8_t *)host_addr; for (i = 0; i < size; i++) *ff_addr++ = 0xff; err = DDI_FAILURE; #ifdef DEBUG pci_peekfault_cnt++; #endif } } else { if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) { for (; repcount; repcount--) { i_ddi_caut_get(size, (void *)dev_addr, (void *)host_addr); host_addr += size; if (flags == DDI_DEV_AUTOINCR) dev_addr += size; } } else { err = DDI_FAILURE; #ifdef DEBUG pci_peekfault_cnt++; #endif } } i_ddi_notrap((ddi_acc_handle_t)hp); pbm_p->pbm_ontrap_data = NULL; i_ndi_busop_access_exit(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp); hp->ahi_err->err_expected = DDI_FM_ERR_UNEXPECTED; return (err); } static int pci_ctlops_peek(pci_t *pci_p, peekpoke_ctlops_t *in_args, void *result) { result = (void *)in_args->host_addr; return (in_args->handle ? pci_do_caut_get(pci_p, in_args) : pci_do_peek(pci_p, in_args)); } /* * get_reg_set_size * * Given a dev info pointer to a pci child and a register number, this * routine returns the size element of that reg set property. * return value: size of reg set on success, -1 on error */ static off_t get_reg_set_size(dev_info_t *child, int rnumber) { pci_regspec_t *pci_rp; off_t size; int i; if (rnumber < 0) return (-1); /* * Get the reg property for the device. */ if (ddi_getlongprop(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS, "reg", (caddr_t)&pci_rp, &i) != DDI_SUCCESS) return (-1); if (rnumber >= (i / (int)sizeof (pci_regspec_t))) { kmem_free(pci_rp, i); return (-1); } size = pci_rp[rnumber].pci_size_low | ((uint64_t)pci_rp[rnumber].pci_size_hi << 32); kmem_free(pci_rp, i); return (size); } /* * control ops entry point: * * Requests handled completely: * DDI_CTLOPS_INITCHILD see init_child() for details * DDI_CTLOPS_UNINITCHILD * DDI_CTLOPS_REPORTDEV see report_dev() for details * DDI_CTLOPS_XLATE_INTRS nothing to do * DDI_CTLOPS_IOMIN cache line size if streaming otherwise 1 * DDI_CTLOPS_REGSIZE * DDI_CTLOPS_NREGS * DDI_CTLOPS_NINTRS * DDI_CTLOPS_DVMAPAGESIZE * DDI_CTLOPS_POKE * DDI_CTLOPS_PEEK * DDI_CTLOPS_QUIESCE * DDI_CTLOPS_UNQUIESCE * * All others passed to parent. */ int pci_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op, void *arg, void *result) { pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip)); switch (op) { case DDI_CTLOPS_INITCHILD: return (init_child(pci_p, (dev_info_t *)arg)); case DDI_CTLOPS_UNINITCHILD: return (uninit_child(pci_p, (dev_info_t *)arg)); case DDI_CTLOPS_REPORTDEV: return (report_dev(rdip)); case DDI_CTLOPS_IOMIN: /* * If we are using the streaming cache, align at * least on a cache line boundary. Otherwise use * whatever alignment is passed in. */ if ((int)arg) { int val = *((int *)result); val = maxbit(val, PCI_SBUF_LINE_SIZE); *((int *)result) = val; } return (DDI_SUCCESS); case DDI_CTLOPS_REGSIZE: *((off_t *)result) = get_reg_set_size(rdip, *((int *)arg)); return (*((off_t *)result) == -1 ? DDI_FAILURE : DDI_SUCCESS); case DDI_CTLOPS_NREGS: *((uint_t *)result) = get_nreg_set(rdip); return (DDI_SUCCESS); case DDI_CTLOPS_DVMAPAGESIZE: *((ulong_t *)result) = IOMMU_PAGE_SIZE; return (DDI_SUCCESS); case DDI_CTLOPS_POKE: return (pci_ctlops_poke(pci_p, (peekpoke_ctlops_t *)arg)); case DDI_CTLOPS_PEEK: return (pci_ctlops_peek(pci_p, (peekpoke_ctlops_t *)arg, result)); case DDI_CTLOPS_AFFINITY: break; case DDI_CTLOPS_QUIESCE: return (pci_bus_quiesce(pci_p, rdip, result)); case DDI_CTLOPS_UNQUIESCE: return (pci_bus_unquiesce(pci_p, rdip, result)); default: break; } /* * Now pass the request up to our parent. */ DEBUG2(DBG_CTLOPS, dip, "passing request to parent: rdip=%s%d\n", ddi_driver_name(rdip), ddi_get_instance(rdip)); return (ddi_ctlops(dip, rdip, op, arg, result)); } /* ARGSUSED */ int pci_intr_ops(dev_info_t *dip, dev_info_t *rdip, ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result) { pci_t *pci_p = get_pci_soft_state( ddi_get_instance(dip)); ddi_ispec_t *ip = (ddi_ispec_t *)hdlp->ih_private; int ret = DDI_SUCCESS; switch (intr_op) { case DDI_INTROP_GETCAP: /* GetCap will always fail for all non PCI devices */ (void) pci_intx_get_cap(rdip, (int *)result); break; case DDI_INTROP_SETCAP: ret = DDI_ENOTSUP; break; case DDI_INTROP_ALLOC: *(int *)result = hdlp->ih_scratch1; break; case DDI_INTROP_FREE: break; case DDI_INTROP_GETPRI: *(int *)result = ip->is_pil ? ip->is_pil : pci_class_to_pil(rdip); break; case DDI_INTROP_SETPRI: ip->is_pil = (*(int *)result); break; case DDI_INTROP_ADDISR: hdlp->ih_vector = *ip->is_intr; ret = pci_add_intr(dip, rdip, hdlp); break; case DDI_INTROP_REMISR: hdlp->ih_vector = *ip->is_intr; ret = pci_remove_intr(dip, rdip, hdlp); break; case DDI_INTROP_ENABLE: ret = ib_update_intr_state(pci_p, rdip, hdlp, PCI_INTR_STATE_ENABLE); break; case DDI_INTROP_DISABLE: ret = ib_update_intr_state(pci_p, rdip, hdlp, PCI_INTR_STATE_DISABLE); break; case DDI_INTROP_SETMASK: ret = pci_intx_set_mask(rdip); break; case DDI_INTROP_CLRMASK: ret = pci_intx_clr_mask(rdip); break; case DDI_INTROP_GETPENDING: ret = pci_intx_get_pending(rdip, (int *)result); break; case DDI_INTROP_NINTRS: case DDI_INTROP_NAVAIL: *(int *)result = i_ddi_get_nintrs(rdip); break; case DDI_INTROP_SUPPORTED_TYPES: /* PCI nexus driver supports only fixed interrupts */ *(int *)result = i_ddi_get_nintrs(rdip) ? DDI_INTR_TYPE_FIXED : 0; break; default: ret = DDI_ENOTSUP; break; } return (ret); } static void pci_init_hotplug(struct pci *pci_p) { pci_bus_range_t bus_range; dev_info_t *dip; struct cb_ops *ops; /* * Before initializing hotplug - open up * bus range. The busra module will * initialize its pool of bus numbers from * this. "busra" will be the agent that keeps * track of them during hotplug. Also, note, * that busra will remove any bus numbers * already in use from boot time. */ bus_range.lo = 0x0; bus_range.hi = 0xff; dip = pci_p->pci_dip; pci_p->hotplug_capable = B_FALSE; /* * If this property exists, this nexus has hot-plug * slots. */ if (ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "hotplug-capable")) { if (ndi_prop_update_int_array(DDI_DEV_T_NONE, dip, "bus-range", (int *)&bus_range, 2) != DDI_PROP_SUCCESS) { return; } if (pcihp_init(dip) != DDI_SUCCESS) { return; } if (ops = pcihp_get_cb_ops()) { pci_ops.devo_cb_ops = ops; DEBUG2(DBG_ATTACH, dip, "%s%d hotplug enabled", ddi_driver_name(dip), ddi_get_instance(dip)); } else { return; } pci_p->hotplug_capable = B_TRUE; } }