1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved. 29 * Copyright 2019 Joyent, Inc. 30 */ 31 32 /* 33 * npe (Nexus PCIe driver): Host to PCI-Express local bus driver 34 * 35 * npe serves as the driver for PCIe Root Complexes and as the nexus driver 36 * for PCIe devices. See also: npe(7D). For more information about hotplug, 37 * see the big theory statement at uts/common/os/ddi_hp_impl.c. 38 * 39 * 40 * NDI EVENT HANDLING SUPPORT 41 * 42 * npe supports NDI event handling. The only available event is surprise 43 * removal of a device. Child drivers can register surprise removal event 44 * callbacks by requesting an event cookie using ddi_get_eventcookie for 45 * the DDI_DEVI_REMOVE_EVENT and add their callback using 46 * ddi_add_event_handler. For an example, see the nvme driver in 47 * uts/common/io/nvme/nvme.c. 48 * 49 * The NDI events in npe are retrieved using NDI_EVENT_NOPASS, which 50 * prevent them from being propagated up the tree once they reach the npe's 51 * bus_get_eventcookie operations. This is important because npe maintains 52 * the state of PCIe devices and their receptacles, via the PCIe hotplug 53 * controller driver (pciehpc). 54 * 55 * Hot removal events are ultimately posted by the PCIe hotplug controller 56 * interrupt handler for hotplug events. Events are posted using the 57 * ndi_post_event interface. 58 */ 59 60 #include <sys/conf.h> 61 #include <sys/modctl.h> 62 #include <sys/file.h> 63 #include <sys/pci_impl.h> 64 #include <sys/pcie_impl.h> 65 #include <sys/sysmacros.h> 66 #include <sys/ddi_intr.h> 67 #include <sys/sunndi.h> 68 #include <sys/sunddi.h> 69 #include <sys/ddifm.h> 70 #include <sys/ndifm.h> 71 #include <sys/fm/util.h> 72 #include <sys/hotplug/pci/pcie_hp.h> 73 #include <io/pci/pci_tools_ext.h> 74 #include <io/pci/pci_common.h> 75 #include <io/pciex/pcie_nvidia.h> 76 77 /* 78 * Helper Macros 79 */ 80 #define NPE_IS_HANDLE_FOR_STDCFG_ACC(hp) \ 81 ((hp) != NULL && \ 82 ((ddi_acc_hdl_t *)(hp))->ah_platform_private != NULL && \ 83 (((ddi_acc_impl_t *)((ddi_acc_hdl_t *)(hp))-> \ 84 ah_platform_private)-> \ 85 ahi_acc_attr &(DDI_ACCATTR_CPU_VADDR|DDI_ACCATTR_CONFIG_SPACE)) \ 86 == DDI_ACCATTR_CONFIG_SPACE) 87 88 /* 89 * Bus Operation functions 90 */ 91 static int npe_bus_map(dev_info_t *, dev_info_t *, ddi_map_req_t *, 92 off_t, off_t, caddr_t *); 93 static int npe_ctlops(dev_info_t *, dev_info_t *, ddi_ctl_enum_t, 94 void *, void *); 95 static int npe_intr_ops(dev_info_t *, dev_info_t *, ddi_intr_op_t, 96 ddi_intr_handle_impl_t *, void *); 97 static int npe_fm_init(dev_info_t *, dev_info_t *, int, 98 ddi_iblock_cookie_t *); 99 static int npe_bus_get_eventcookie(dev_info_t *, dev_info_t *, char *, 100 ddi_eventcookie_t *); 101 static int npe_bus_add_eventcall(dev_info_t *, dev_info_t *, 102 ddi_eventcookie_t, void (*)(dev_info_t *, 103 ddi_eventcookie_t, void *, void *), 104 void *, ddi_callback_id_t *); 105 static int npe_bus_remove_eventcall(dev_info_t *, ddi_callback_id_t); 106 static int npe_bus_post_event(dev_info_t *, dev_info_t *, 107 ddi_eventcookie_t, void *); 108 109 static int npe_fm_callback(dev_info_t *, ddi_fm_error_t *, const void *); 110 111 /* 112 * Disable URs and Received MA for all PCIe devices. Until x86 SW is changed so 113 * that random drivers do not do PIO accesses on devices that it does not own, 114 * these error bits must be disabled. SERR must also be disabled if URs have 115 * been masked. 116 */ 117 uint32_t npe_aer_uce_mask = PCIE_AER_UCE_UR; 118 uint32_t npe_aer_ce_mask = 0; 119 uint32_t npe_aer_suce_mask = PCIE_AER_SUCE_RCVD_MA; 120 121 struct bus_ops npe_bus_ops = { 122 BUSO_REV, 123 npe_bus_map, 124 NULL, 125 NULL, 126 NULL, 127 i_ddi_map_fault, 128 NULL, 129 ddi_dma_allochdl, 130 ddi_dma_freehdl, 131 ddi_dma_bindhdl, 132 ddi_dma_unbindhdl, 133 ddi_dma_flush, 134 ddi_dma_win, 135 ddi_dma_mctl, 136 npe_ctlops, 137 ddi_bus_prop_op, 138 npe_bus_get_eventcookie, 139 npe_bus_add_eventcall, 140 npe_bus_remove_eventcall, 141 npe_bus_post_event, 142 0, /* (*bus_intr_ctl)(); */ 143 0, /* (*bus_config)(); */ 144 0, /* (*bus_unconfig)(); */ 145 npe_fm_init, /* (*bus_fm_init)(); */ 146 NULL, /* (*bus_fm_fini)(); */ 147 NULL, /* (*bus_fm_access_enter)(); */ 148 NULL, /* (*bus_fm_access_exit)(); */ 149 NULL, /* (*bus_power)(); */ 150 npe_intr_ops, /* (*bus_intr_op)(); */ 151 pcie_hp_common_ops /* (*bus_hp_op)(); */ 152 }; 153 154 static int npe_open(dev_t *, int, int, cred_t *); 155 static int npe_close(dev_t, int, int, cred_t *); 156 static int npe_ioctl(dev_t, int, intptr_t, int, cred_t *, int *); 157 158 struct cb_ops npe_cb_ops = { 159 npe_open, /* open */ 160 npe_close, /* close */ 161 nodev, /* strategy */ 162 nodev, /* print */ 163 nodev, /* dump */ 164 nodev, /* read */ 165 nodev, /* write */ 166 npe_ioctl, /* ioctl */ 167 nodev, /* devmap */ 168 nodev, /* mmap */ 169 nodev, /* segmap */ 170 nochpoll, /* poll */ 171 pcie_prop_op, /* cb_prop_op */ 172 NULL, /* streamtab */ 173 D_NEW | D_MP | D_HOTPLUG, /* Driver compatibility flag */ 174 CB_REV, /* rev */ 175 nodev, /* int (*cb_aread)() */ 176 nodev /* int (*cb_awrite)() */ 177 }; 178 179 180 /* 181 * Device Node Operation functions 182 */ 183 static int npe_attach(dev_info_t *devi, ddi_attach_cmd_t cmd); 184 static int npe_detach(dev_info_t *devi, ddi_detach_cmd_t cmd); 185 static int npe_info(dev_info_t *, ddi_info_cmd_t, void *, void **); 186 187 struct dev_ops npe_ops = { 188 DEVO_REV, /* devo_rev */ 189 0, /* refcnt */ 190 npe_info, /* info */ 191 nulldev, /* identify */ 192 nulldev, /* probe */ 193 npe_attach, /* attach */ 194 npe_detach, /* detach */ 195 nulldev, /* reset */ 196 &npe_cb_ops, /* driver operations */ 197 &npe_bus_ops, /* bus operations */ 198 NULL, /* power */ 199 ddi_quiesce_not_needed, /* quiesce */ 200 }; 201 202 /* 203 * Internal routines in support of particular npe_ctlops. 204 */ 205 static int npe_removechild(dev_info_t *child); 206 static int npe_initchild(dev_info_t *child); 207 208 /* 209 * External support routine 210 */ 211 extern void npe_query_acpi_mcfg(dev_info_t *dip); 212 extern void npe_ck804_fix_aer_ptr(ddi_acc_handle_t cfg_hdl); 213 extern int npe_disable_empty_bridges_workaround(dev_info_t *child); 214 extern void npe_nvidia_error_workaround(ddi_acc_handle_t cfg_hdl); 215 extern void npe_intel_error_workaround(ddi_acc_handle_t cfg_hdl); 216 extern boolean_t npe_is_mmcfg_supported(dev_info_t *dip); 217 extern void npe_enable_htmsi_children(dev_info_t *dip); 218 extern int npe_save_htconfig_children(dev_info_t *dip); 219 extern int npe_restore_htconfig_children(dev_info_t *dip); 220 221 /* 222 * Module linkage information for the kernel. 223 */ 224 static struct modldrv modldrv = { 225 &mod_driverops, /* Type of module */ 226 "Host to PCIe nexus driver", /* Name of module */ 227 &npe_ops, /* driver ops */ 228 }; 229 230 static struct modlinkage modlinkage = { 231 MODREV_1, 232 (void *)&modldrv, 233 NULL 234 }; 235 236 /* Save minimal state. */ 237 void *npe_statep; 238 239 int 240 _init(void) 241 { 242 int e; 243 244 /* 245 * Initialize per-pci bus soft state pointer. 246 */ 247 e = ddi_soft_state_init(&npe_statep, sizeof (pci_state_t), 1); 248 if (e != 0) 249 return (e); 250 251 if ((e = mod_install(&modlinkage)) != 0) 252 ddi_soft_state_fini(&npe_statep); 253 254 return (e); 255 } 256 257 258 int 259 _fini(void) 260 { 261 int rc; 262 263 rc = mod_remove(&modlinkage); 264 if (rc != 0) 265 return (rc); 266 267 ddi_soft_state_fini(&npe_statep); 268 return (rc); 269 } 270 271 272 int 273 _info(struct modinfo *modinfop) 274 { 275 return (mod_info(&modlinkage, modinfop)); 276 } 277 278 /*ARGSUSED*/ 279 static int 280 npe_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) 281 { 282 minor_t minor = getminor((dev_t)arg); 283 int instance = PCI_MINOR_NUM_TO_INSTANCE(minor); 284 pci_state_t *pcip = ddi_get_soft_state(npe_statep, instance); 285 int ret = DDI_SUCCESS; 286 287 switch (cmd) { 288 case DDI_INFO_DEVT2INSTANCE: 289 *result = (void *)(intptr_t)instance; 290 break; 291 case DDI_INFO_DEVT2DEVINFO: 292 if (pcip == NULL) { 293 ret = DDI_FAILURE; 294 break; 295 } 296 297 *result = (void *)pcip->pci_dip; 298 break; 299 default: 300 ret = DDI_FAILURE; 301 break; 302 } 303 304 return (ret); 305 } 306 307 /* 308 * See big theory statement at the top of this file for more information about 309 * surprise removal events. 310 */ 311 #define NPE_EVENT_TAG_HOT_REMOVAL 0 312 static ndi_event_definition_t npe_ndi_event_defs[1] = { 313 {NPE_EVENT_TAG_HOT_REMOVAL, DDI_DEVI_REMOVE_EVENT, EPL_KERNEL, 314 NDI_EVENT_POST_TO_ALL} 315 }; 316 317 static ndi_event_set_t npe_ndi_events = { 318 NDI_EVENTS_REV1, ARRAY_SIZE(npe_ndi_event_defs), npe_ndi_event_defs 319 }; 320 321 /*ARGSUSED*/ 322 static int 323 npe_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) 324 { 325 int instance = ddi_get_instance(devi); 326 pci_state_t *pcip = NULL; 327 int ret; 328 329 if (cmd == DDI_RESUME) { 330 /* 331 * the system might still be able to resume even if this fails 332 */ 333 (void) npe_restore_htconfig_children(devi); 334 return (DDI_SUCCESS); 335 } 336 337 /* 338 * We must do this here in order to ensure that all top level devices 339 * get their HyperTransport MSI mapping regs programmed first. 340 * "Memory controller" and "hostbridge" class devices are leaf devices 341 * that may affect MSI translation functionality for devices 342 * connected to the same link/bus. 343 * 344 * This will also program HT MSI mapping registers on root buses 345 * devices (basically sitting on an HT bus) that are not dependent 346 * on the aforementioned HT devices for MSI translation. 347 */ 348 npe_enable_htmsi_children(devi); 349 350 if (ddi_prop_update_string(DDI_DEV_T_NONE, devi, "device_type", 351 "pciex") != DDI_PROP_SUCCESS) { 352 cmn_err(CE_WARN, "npe: 'device_type' prop create failed"); 353 } 354 355 if (ddi_soft_state_zalloc(npe_statep, instance) == DDI_SUCCESS) 356 pcip = ddi_get_soft_state(npe_statep, instance); 357 358 if (pcip == NULL) 359 return (DDI_FAILURE); 360 361 pcip->pci_dip = devi; 362 pcip->pci_soft_state = PCI_SOFT_STATE_CLOSED; 363 364 if (pcie_init(devi, NULL) != DDI_SUCCESS) 365 goto fail1; 366 367 ret = ndi_event_alloc_hdl(pcip->pci_dip, NULL, &pcip->pci_ndi_event_hdl, 368 NDI_SLEEP); 369 if (ret == NDI_SUCCESS) { 370 ret = ndi_event_bind_set(pcip->pci_ndi_event_hdl, 371 &npe_ndi_events, NDI_SLEEP); 372 if (ret != NDI_SUCCESS) { 373 dev_err(pcip->pci_dip, CE_WARN, "npe: failed to bind " 374 "NDI event set (error=%d)", ret); 375 goto fail1; 376 } 377 } else { 378 dev_err(pcip->pci_dip, CE_WARN, "npe: failed to allocate " 379 "event handle (error=%d)", ret); 380 goto fail1; 381 } 382 383 /* Second arg: initialize for pci_express root nexus */ 384 if (pcitool_init(devi, B_TRUE) != DDI_SUCCESS) 385 goto fail2; 386 387 pcip->pci_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 388 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 389 ddi_fm_init(devi, &pcip->pci_fmcap, &pcip->pci_fm_ibc); 390 391 if (pcip->pci_fmcap & DDI_FM_ERRCB_CAPABLE) { 392 ddi_fm_handler_register(devi, npe_fm_callback, NULL); 393 } 394 395 PCIE_DIP2PFD(devi) = kmem_zalloc(sizeof (pf_data_t), KM_SLEEP); 396 pcie_rc_init_pfd(devi, PCIE_DIP2PFD(devi)); 397 398 npe_query_acpi_mcfg(devi); 399 ddi_report_dev(devi); 400 pcie_fab_init_bus(devi, PCIE_BUS_FINAL); 401 402 return (DDI_SUCCESS); 403 404 fail2: 405 (void) pcie_uninit(devi); 406 fail1: 407 pcie_rc_fini_bus(devi); 408 ddi_soft_state_free(npe_statep, instance); 409 410 return (DDI_FAILURE); 411 } 412 413 /*ARGSUSED*/ 414 static int 415 npe_detach(dev_info_t *devi, ddi_detach_cmd_t cmd) 416 { 417 int instance = ddi_get_instance(devi); 418 pci_state_t *pcip; 419 int ret; 420 421 pcip = ddi_get_soft_state(npe_statep, ddi_get_instance(devi)); 422 423 switch (cmd) { 424 case DDI_DETACH: 425 426 /* 427 * Clean up event handling first, to ensure there are no 428 * oustanding callbacks registered. 429 */ 430 ret = ndi_event_unbind_set(pcip->pci_ndi_event_hdl, 431 &npe_ndi_events, NDI_SLEEP); 432 if (ret == NDI_SUCCESS) { 433 /* ndi_event_free_hdl always succeeds. */ 434 (void) ndi_event_free_hdl(pcip->pci_ndi_event_hdl); 435 } else { 436 /* 437 * The event set will only fail to unbind if there are 438 * outstanding callbacks registered for it, which 439 * probably means a child driver still has one 440 * registered and thus was not cleaned up properly 441 * before npe's detach routine was called. Consequently, 442 * we should fail the detach here. 443 */ 444 dev_err(pcip->pci_dip, CE_WARN, "npe: failed to " 445 "unbind NDI event set (error=%d)", ret); 446 return (DDI_FAILURE); 447 } 448 449 pcie_fab_fini_bus(devi, PCIE_BUS_INITIAL); 450 451 /* Uninitialize pcitool support. */ 452 pcitool_uninit(devi); 453 454 if (pcie_uninit(devi) != DDI_SUCCESS) 455 return (DDI_FAILURE); 456 457 if (pcip->pci_fmcap & DDI_FM_ERRCB_CAPABLE) 458 ddi_fm_handler_unregister(devi); 459 460 pcie_rc_fini_pfd(PCIE_DIP2PFD(devi)); 461 kmem_free(PCIE_DIP2PFD(devi), sizeof (pf_data_t)); 462 463 ddi_fm_fini(devi); 464 ddi_soft_state_free(npe_statep, instance); 465 466 return (DDI_SUCCESS); 467 468 case DDI_SUSPEND: 469 /* 470 * the system might still be able to suspend/resume even if 471 * this fails 472 */ 473 (void) npe_save_htconfig_children(devi); 474 return (DDI_SUCCESS); 475 default: 476 return (DDI_FAILURE); 477 } 478 } 479 480 /* 481 * Configure the access handle for standard configuration space 482 * access (see pci_fm_acc_setup for code that initializes the 483 * access-function pointers). 484 */ 485 static int 486 npe_setup_std_pcicfg_acc(dev_info_t *rdip, ddi_map_req_t *mp, 487 ddi_acc_hdl_t *hp, off_t offset, off_t len) 488 { 489 int ret; 490 491 if ((ret = pci_fm_acc_setup(hp, offset, len)) == 492 DDI_SUCCESS) { 493 if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) && 494 mp->map_handlep->ah_acc.devacc_attr_access 495 != DDI_DEFAULT_ACC) { 496 ndi_fmc_insert(rdip, ACC_HANDLE, 497 (void *)mp->map_handlep, NULL); 498 } 499 } 500 return (ret); 501 } 502 503 static int 504 npe_bus_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 505 off_t offset, off_t len, caddr_t *vaddrp) 506 { 507 int rnumber; 508 int space; 509 ddi_acc_impl_t *ap; 510 ddi_acc_hdl_t *hp; 511 ddi_map_req_t mr; 512 pci_regspec_t pci_reg; 513 pci_regspec_t *pci_rp; 514 struct regspec64 reg; 515 pci_acc_cfblk_t *cfp; 516 int retval; 517 int64_t *ecfginfo; 518 uint_t nelem; 519 uint64_t pci_rlength; 520 521 mr = *mp; /* Get private copy of request */ 522 mp = &mr; 523 524 /* 525 * check for register number 526 */ 527 switch (mp->map_type) { 528 case DDI_MT_REGSPEC: 529 pci_reg = *(pci_regspec_t *)(mp->map_obj.rp); 530 pci_rp = &pci_reg; 531 if (pci_common_get_reg_prop(rdip, pci_rp) != DDI_SUCCESS) 532 return (DDI_FAILURE); 533 break; 534 case DDI_MT_RNUMBER: 535 rnumber = mp->map_obj.rnumber; 536 /* 537 * get ALL "reg" properties for dip, select the one of 538 * of interest. In x86, "assigned-addresses" property 539 * is identical to the "reg" property, so there is no 540 * need to cross check the two to determine the physical 541 * address of the registers. 542 * This routine still performs some validity checks to 543 * make sure that everything is okay. 544 */ 545 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, rdip, 546 DDI_PROP_DONTPASS, "reg", (int **)&pci_rp, &nelem) != 547 DDI_PROP_SUCCESS) 548 return (DDI_FAILURE); 549 550 /* 551 * validate the register number. 552 */ 553 nelem /= (sizeof (pci_regspec_t) / sizeof (int)); 554 if (rnumber >= nelem) { 555 ddi_prop_free(pci_rp); 556 return (DDI_FAILURE); 557 } 558 559 /* 560 * copy the required entry. 561 */ 562 pci_reg = pci_rp[rnumber]; 563 564 /* 565 * free the memory allocated by ddi_prop_lookup_int_array 566 */ 567 ddi_prop_free(pci_rp); 568 569 pci_rp = &pci_reg; 570 if (pci_common_get_reg_prop(rdip, pci_rp) != DDI_SUCCESS) 571 return (DDI_FAILURE); 572 mp->map_type = DDI_MT_REGSPEC; 573 break; 574 default: 575 return (DDI_ME_INVAL); 576 } 577 578 space = pci_rp->pci_phys_hi & PCI_REG_ADDR_M; 579 580 /* 581 * check for unmap and unlock of address space 582 */ 583 if ((mp->map_op == DDI_MO_UNMAP) || (mp->map_op == DDI_MO_UNLOCK)) { 584 switch (space) { 585 case PCI_ADDR_IO: 586 reg.regspec_bustype = 1; 587 break; 588 589 case PCI_ADDR_CONFIG: 590 /* 591 * If this is an unmap/unlock of a standard config 592 * space mapping (memory-mapped config space mappings 593 * would have the DDI_ACCATTR_CPU_VADDR bit set in the 594 * acc_attr), undo that setup here. 595 */ 596 if (NPE_IS_HANDLE_FOR_STDCFG_ACC(mp->map_handlep)) { 597 598 if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) && 599 mp->map_handlep->ah_acc.devacc_attr_access 600 != DDI_DEFAULT_ACC) { 601 ndi_fmc_remove(rdip, ACC_HANDLE, 602 (void *)mp->map_handlep); 603 } 604 return (DDI_SUCCESS); 605 } 606 607 pci_rp->pci_size_low = PCIE_CONF_HDR_SIZE; 608 609 /* FALLTHROUGH */ 610 case PCI_ADDR_MEM64: 611 case PCI_ADDR_MEM32: 612 reg.regspec_bustype = 0; 613 break; 614 615 default: 616 return (DDI_FAILURE); 617 } 618 619 reg.regspec_addr = (uint64_t)pci_rp->pci_phys_mid << 32 | 620 (uint64_t)pci_rp->pci_phys_low; 621 reg.regspec_size = (uint64_t)pci_rp->pci_size_hi << 32 | 622 (uint64_t)pci_rp->pci_size_low; 623 624 /* 625 * Adjust offset and length 626 * A non-zero length means override the one in the regspec. 627 */ 628 if (reg.regspec_addr + offset < MAX(reg.regspec_addr, offset)) 629 return (DDI_FAILURE); 630 reg.regspec_addr += offset; 631 if (len != 0) 632 reg.regspec_size = len; 633 634 mp->map_obj.rp = (struct regspec *)® 635 mp->map_flags |= DDI_MF_EXT_REGSPEC; 636 retval = ddi_map(dip, mp, (off_t)0, (off_t)0, vaddrp); 637 if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) && 638 mp->map_handlep->ah_acc.devacc_attr_access != 639 DDI_DEFAULT_ACC) { 640 ndi_fmc_remove(rdip, ACC_HANDLE, 641 (void *)mp->map_handlep); 642 } 643 return (retval); 644 645 } 646 647 /* check for user mapping request - not legal for Config */ 648 if (mp->map_op == DDI_MO_MAP_HANDLE && space == PCI_ADDR_CONFIG) { 649 cmn_err(CE_NOTE, "npe: Config mapping request from user\n"); 650 return (DDI_FAILURE); 651 } 652 653 654 /* 655 * Note that pci_fm_acc_setup() is called to serve two purposes 656 * i) enable legacy PCI I/O style config space access 657 * ii) register with FMA 658 */ 659 if (space == PCI_ADDR_CONFIG) { 660 661 /* Can't map config space without a handle */ 662 hp = (ddi_acc_hdl_t *)mp->map_handlep; 663 if (hp == NULL) 664 return (DDI_FAILURE); 665 666 /* record the device address for future reference */ 667 cfp = (pci_acc_cfblk_t *)&hp->ah_bus_private; 668 cfp->c_busnum = PCI_REG_BUS_G(pci_rp->pci_phys_hi); 669 cfp->c_devnum = PCI_REG_DEV_G(pci_rp->pci_phys_hi); 670 cfp->c_funcnum = PCI_REG_FUNC_G(pci_rp->pci_phys_hi); 671 672 *vaddrp = (caddr_t)offset; 673 674 /* Check if MMCFG is supported */ 675 if (!npe_is_mmcfg_supported(rdip)) { 676 return (npe_setup_std_pcicfg_acc(rdip, mp, hp, 677 offset, len)); 678 } 679 680 681 if (ddi_prop_lookup_int64_array(DDI_DEV_T_ANY, rdip, 0, 682 "ecfg", &ecfginfo, &nelem) == DDI_PROP_SUCCESS) { 683 684 if (nelem != 4 || 685 cfp->c_busnum < ecfginfo[2] || 686 cfp->c_busnum > ecfginfo[3]) { 687 /* 688 * Invalid property or Doesn't contain the 689 * requested bus; fall back to standard 690 * (I/O-based) config access. 691 */ 692 ddi_prop_free(ecfginfo); 693 return (npe_setup_std_pcicfg_acc(rdip, mp, hp, 694 offset, len)); 695 } else { 696 pci_rp->pci_phys_low = ecfginfo[0]; 697 698 ddi_prop_free(ecfginfo); 699 700 pci_rp->pci_phys_low += ((cfp->c_busnum << 20) | 701 (cfp->c_devnum) << 15 | 702 (cfp->c_funcnum << 12)); 703 704 pci_rp->pci_size_low = PCIE_CONF_HDR_SIZE; 705 } 706 } else { 707 /* 708 * Couldn't find the MMCFG property -- fall back to 709 * standard config access 710 */ 711 return (npe_setup_std_pcicfg_acc(rdip, mp, hp, 712 offset, len)); 713 } 714 } 715 716 /* 717 * range check 718 */ 719 pci_rlength = (uint64_t)pci_rp->pci_size_low | 720 (uint64_t)pci_rp->pci_size_hi << 32; 721 if ((offset >= pci_rlength) || (len > pci_rlength) || 722 (offset + len > pci_rlength) || (offset + len < MAX(offset, len))) { 723 return (DDI_FAILURE); 724 } 725 726 /* 727 * convert the pci regsec into the generic regspec used by the 728 * parent root nexus driver. 729 */ 730 switch (space) { 731 case PCI_ADDR_IO: 732 reg.regspec_bustype = 1; 733 break; 734 case PCI_ADDR_CONFIG: 735 case PCI_ADDR_MEM64: 736 case PCI_ADDR_MEM32: 737 reg.regspec_bustype = 0; 738 break; 739 default: 740 return (DDI_FAILURE); 741 } 742 743 reg.regspec_addr = (uint64_t)pci_rp->pci_phys_mid << 32 | 744 (uint64_t)pci_rp->pci_phys_low; 745 reg.regspec_size = pci_rlength; 746 747 /* 748 * Adjust offset and length 749 * A non-zero length means override the one in the regspec. 750 */ 751 if (reg.regspec_addr + offset < MAX(reg.regspec_addr, offset)) 752 return (DDI_FAILURE); 753 reg.regspec_addr += offset; 754 if (len != 0) 755 reg.regspec_size = len; 756 757 758 mp->map_obj.rp = (struct regspec *)® 759 mp->map_flags |= DDI_MF_EXT_REGSPEC; 760 retval = ddi_map(dip, mp, (off_t)0, (off_t)0, vaddrp); 761 if (retval == DDI_SUCCESS) { 762 /* 763 * For config space gets force use of cautious access routines. 764 * These will handle default and protected mode accesses too. 765 */ 766 if (space == PCI_ADDR_CONFIG) { 767 ap = (ddi_acc_impl_t *)mp->map_handlep; 768 ap->ahi_acc_attr &= ~DDI_ACCATTR_DIRECT; 769 ap->ahi_acc_attr |= DDI_ACCATTR_CONFIG_SPACE; 770 ap->ahi_get8 = i_ddi_caut_get8; 771 ap->ahi_get16 = i_ddi_caut_get16; 772 ap->ahi_get32 = i_ddi_caut_get32; 773 ap->ahi_get64 = i_ddi_caut_get64; 774 ap->ahi_rep_get8 = i_ddi_caut_rep_get8; 775 ap->ahi_rep_get16 = i_ddi_caut_rep_get16; 776 ap->ahi_rep_get32 = i_ddi_caut_rep_get32; 777 ap->ahi_rep_get64 = i_ddi_caut_rep_get64; 778 } 779 if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) && 780 mp->map_handlep->ah_acc.devacc_attr_access != 781 DDI_DEFAULT_ACC) { 782 ndi_fmc_insert(rdip, ACC_HANDLE, 783 (void *)mp->map_handlep, NULL); 784 } 785 } 786 return (retval); 787 } 788 789 790 791 /*ARGSUSED*/ 792 static int 793 npe_ctlops(dev_info_t *dip, dev_info_t *rdip, 794 ddi_ctl_enum_t ctlop, void *arg, void *result) 795 { 796 int totreg; 797 uint_t reglen; 798 pci_regspec_t *drv_regp; 799 struct attachspec *asp; 800 struct detachspec *dsp; 801 pci_state_t *pci_p = ddi_get_soft_state(npe_statep, 802 ddi_get_instance(dip)); 803 804 switch (ctlop) { 805 case DDI_CTLOPS_REPORTDEV: 806 if (rdip == (dev_info_t *)0) 807 return (DDI_FAILURE); 808 cmn_err(CE_CONT, "?PCI Express-device: %s@%s, %s%d\n", 809 ddi_node_name(rdip), ddi_get_name_addr(rdip), 810 ddi_driver_name(rdip), ddi_get_instance(rdip)); 811 return (DDI_SUCCESS); 812 813 case DDI_CTLOPS_INITCHILD: 814 return (npe_initchild((dev_info_t *)arg)); 815 816 case DDI_CTLOPS_UNINITCHILD: 817 return (npe_removechild((dev_info_t *)arg)); 818 819 case DDI_CTLOPS_SIDDEV: 820 return (DDI_SUCCESS); 821 822 case DDI_CTLOPS_REGSIZE: 823 case DDI_CTLOPS_NREGS: 824 if (rdip == (dev_info_t *)0) 825 return (DDI_FAILURE); 826 827 *(int *)result = 0; 828 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, rdip, 829 DDI_PROP_DONTPASS, "reg", (int **)&drv_regp, 830 ®len) != DDI_PROP_SUCCESS) { 831 return (DDI_FAILURE); 832 } 833 834 totreg = (reglen * sizeof (int)) / sizeof (pci_regspec_t); 835 if (ctlop == DDI_CTLOPS_NREGS) 836 *(int *)result = totreg; 837 else if (ctlop == DDI_CTLOPS_REGSIZE) { 838 uint64_t val; 839 int rn; 840 841 rn = *(int *)arg; 842 if (rn >= totreg) { 843 ddi_prop_free(drv_regp); 844 return (DDI_FAILURE); 845 } 846 val = drv_regp[rn].pci_size_low | 847 (uint64_t)drv_regp[rn].pci_size_hi << 32; 848 if (val > OFF_MAX) { 849 int ce = CE_NOTE; 850 #ifdef DEBUG 851 ce = CE_WARN; 852 #endif 853 dev_err(rdip, ce, "failed to get register " 854 "size, value larger than OFF_MAX: 0x%" 855 PRIx64 "\n", val); 856 return (DDI_FAILURE); 857 } 858 *(off_t *)result = (off_t)val; 859 } 860 ddi_prop_free(drv_regp); 861 862 return (DDI_SUCCESS); 863 864 case DDI_CTLOPS_POWER: 865 { 866 power_req_t *reqp = (power_req_t *)arg; 867 /* 868 * We currently understand reporting of PCI_PM_IDLESPEED 869 * capability. Everything else is passed up. 870 */ 871 if ((reqp->request_type == PMR_REPORT_PMCAP) && 872 (reqp->req.report_pmcap_req.cap == PCI_PM_IDLESPEED)) 873 return (DDI_SUCCESS); 874 875 break; 876 } 877 878 case DDI_CTLOPS_PEEK: 879 case DDI_CTLOPS_POKE: 880 return (pci_common_peekpoke(dip, rdip, ctlop, arg, result)); 881 882 /* X86 systems support PME wakeup from suspended state */ 883 case DDI_CTLOPS_ATTACH: 884 if (!pcie_is_child(dip, rdip)) 885 return (DDI_SUCCESS); 886 887 asp = (struct attachspec *)arg; 888 if ((asp->when == DDI_POST) && (asp->result == DDI_SUCCESS)) { 889 pf_init(rdip, (void *)pci_p->pci_fm_ibc, asp->cmd); 890 (void) pcie_postattach_child(rdip); 891 } 892 893 /* only do this for immediate children */ 894 if (asp->cmd == DDI_RESUME && asp->when == DDI_PRE && 895 ddi_get_parent(rdip) == dip) 896 if (pci_pre_resume(rdip) != DDI_SUCCESS) { 897 /* Not good, better stop now. */ 898 cmn_err(CE_PANIC, 899 "Couldn't pre-resume device %p", 900 (void *) dip); 901 /* NOTREACHED */ 902 } 903 904 return (DDI_SUCCESS); 905 906 case DDI_CTLOPS_DETACH: 907 if (!pcie_is_child(dip, rdip)) 908 return (DDI_SUCCESS); 909 910 dsp = (struct detachspec *)arg; 911 912 if (dsp->when == DDI_PRE) 913 pf_fini(rdip, dsp->cmd); 914 915 /* only do this for immediate children */ 916 if (dsp->cmd == DDI_SUSPEND && dsp->when == DDI_POST && 917 ddi_get_parent(rdip) == dip) 918 if (pci_post_suspend(rdip) != DDI_SUCCESS) 919 return (DDI_FAILURE); 920 921 return (DDI_SUCCESS); 922 923 default: 924 break; 925 } 926 927 return (ddi_ctlops(dip, rdip, ctlop, arg, result)); 928 929 } 930 931 932 /* 933 * npe_intr_ops 934 */ 935 static int 936 npe_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 937 ddi_intr_handle_impl_t *hdlp, void *result) 938 { 939 return (pci_common_intr_ops(pdip, rdip, intr_op, hdlp, result)); 940 } 941 942 943 static int 944 npe_initchild(dev_info_t *child) 945 { 946 char name[80]; 947 pcie_bus_t *bus_p; 948 uint32_t regs; 949 ddi_acc_handle_t cfg_hdl; 950 951 /* 952 * Do not bind drivers to empty bridges. 953 * Fail above, if the bridge is found to be hotplug capable 954 */ 955 if (npe_disable_empty_bridges_workaround(child) == 1) 956 return (DDI_FAILURE); 957 958 if (pci_common_name_child(child, name, 80) != DDI_SUCCESS) 959 return (DDI_FAILURE); 960 961 ddi_set_name_addr(child, name); 962 963 /* 964 * Pseudo nodes indicate a prototype node with per-instance 965 * properties to be merged into the real h/w device node. 966 * The interpretation of the unit-address is DD[,F] 967 * where DD is the device id and F is the function. 968 */ 969 if (ndi_dev_is_persistent_node(child) == 0) { 970 extern int pci_allow_pseudo_children; 971 972 ddi_set_parent_data(child, NULL); 973 974 /* 975 * Try to merge the properties from this prototype 976 * node into real h/w nodes. 977 */ 978 if (ndi_merge_node(child, pci_common_name_child) == 979 DDI_SUCCESS) { 980 /* 981 * Merged ok - return failure to remove the node. 982 */ 983 ddi_set_name_addr(child, NULL); 984 return (DDI_FAILURE); 985 } 986 987 /* workaround for DDIVS to run under PCI Express */ 988 if (pci_allow_pseudo_children) { 989 /* 990 * If the "interrupts" property doesn't exist, 991 * this must be the ddivs no-intr case, and it returns 992 * DDI_SUCCESS instead of DDI_FAILURE. 993 */ 994 if (ddi_prop_get_int(DDI_DEV_T_ANY, child, 995 DDI_PROP_DONTPASS, "interrupts", -1) == -1) 996 return (DDI_SUCCESS); 997 /* 998 * Create the ddi_parent_private_data for a pseudo 999 * child. 1000 */ 1001 pci_common_set_parent_private_data(child); 1002 return (DDI_SUCCESS); 1003 } 1004 1005 /* 1006 * The child was not merged into a h/w node, 1007 * but there's not much we can do with it other 1008 * than return failure to cause the node to be removed. 1009 */ 1010 cmn_err(CE_WARN, "!%s@%s: %s.conf properties not merged", 1011 ddi_get_name(child), ddi_get_name_addr(child), 1012 ddi_get_name(child)); 1013 ddi_set_name_addr(child, NULL); 1014 return (DDI_NOT_WELL_FORMED); 1015 } 1016 1017 if (ddi_prop_get_int(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS, 1018 "interrupts", -1) != -1) 1019 pci_common_set_parent_private_data(child); 1020 else 1021 ddi_set_parent_data(child, NULL); 1022 1023 /* Disable certain errors on PCIe drivers for x86 platforms */ 1024 regs = pcie_get_aer_uce_mask() | npe_aer_uce_mask; 1025 pcie_set_aer_uce_mask(regs); 1026 regs = pcie_get_aer_ce_mask() | npe_aer_ce_mask; 1027 pcie_set_aer_ce_mask(regs); 1028 regs = pcie_get_aer_suce_mask() | npe_aer_suce_mask; 1029 pcie_set_aer_suce_mask(regs); 1030 1031 /* 1032 * If URs are disabled, mask SERRs as well, otherwise the system will 1033 * still be notified of URs 1034 */ 1035 if (npe_aer_uce_mask & PCIE_AER_UCE_UR) 1036 pcie_set_serr_mask(1); 1037 1038 if (pci_config_setup(child, &cfg_hdl) == DDI_SUCCESS) { 1039 npe_ck804_fix_aer_ptr(cfg_hdl); 1040 npe_nvidia_error_workaround(cfg_hdl); 1041 npe_intel_error_workaround(cfg_hdl); 1042 pci_config_teardown(&cfg_hdl); 1043 } 1044 1045 bus_p = PCIE_DIP2BUS(child); 1046 if (bus_p) { 1047 uint16_t device_id = (uint16_t)(bus_p->bus_dev_ven_id >> 16); 1048 uint16_t vendor_id = (uint16_t)(bus_p->bus_dev_ven_id & 0xFFFF); 1049 uint16_t rev_id = bus_p->bus_rev_id; 1050 1051 /* Disable AER for certain NVIDIA Chipsets */ 1052 if ((vendor_id == NVIDIA_VENDOR_ID) && 1053 (device_id == NVIDIA_CK804_DEVICE_ID) && 1054 (rev_id < NVIDIA_CK804_AER_VALID_REVID)) 1055 bus_p->bus_aer_off = 0; 1056 1057 pcie_init_dom(child); 1058 (void) pcie_initchild(child); 1059 } 1060 1061 return (DDI_SUCCESS); 1062 } 1063 1064 1065 static int 1066 npe_removechild(dev_info_t *dip) 1067 { 1068 pcie_uninitchild(dip); 1069 1070 ddi_set_name_addr(dip, NULL); 1071 1072 /* 1073 * Strip the node to properly convert it back to prototype form 1074 */ 1075 ddi_remove_minor_node(dip, NULL); 1076 1077 ddi_prop_remove_all(dip); 1078 1079 return (DDI_SUCCESS); 1080 } 1081 1082 static int 1083 npe_open(dev_t *devp, int flags, int otyp, cred_t *credp) 1084 { 1085 minor_t minor = getminor(*devp); 1086 int instance = PCI_MINOR_NUM_TO_INSTANCE(minor); 1087 pci_state_t *pci_p = ddi_get_soft_state(npe_statep, instance); 1088 int rv; 1089 1090 /* 1091 * Make sure the open is for the right file type. 1092 */ 1093 if (otyp != OTYP_CHR) 1094 return (EINVAL); 1095 1096 if (pci_p == NULL) 1097 return (ENXIO); 1098 1099 mutex_enter(&pci_p->pci_mutex); 1100 switch (PCI_MINOR_NUM_TO_PCI_DEVNUM(minor)) { 1101 case PCI_TOOL_REG_MINOR_NUM: 1102 case PCI_TOOL_INTR_MINOR_NUM: 1103 break; 1104 default: 1105 /* Handle devctl ioctls */ 1106 rv = pcie_open(pci_p->pci_dip, devp, flags, otyp, credp); 1107 mutex_exit(&pci_p->pci_mutex); 1108 return (rv); 1109 } 1110 1111 /* Handle pcitool ioctls */ 1112 if (flags & FEXCL) { 1113 if (pci_p->pci_soft_state != PCI_SOFT_STATE_CLOSED) { 1114 mutex_exit(&pci_p->pci_mutex); 1115 cmn_err(CE_NOTE, "npe_open: busy"); 1116 return (EBUSY); 1117 } 1118 pci_p->pci_soft_state = PCI_SOFT_STATE_OPEN_EXCL; 1119 } else { 1120 if (pci_p->pci_soft_state == PCI_SOFT_STATE_OPEN_EXCL) { 1121 mutex_exit(&pci_p->pci_mutex); 1122 cmn_err(CE_NOTE, "npe_open: busy"); 1123 return (EBUSY); 1124 } 1125 pci_p->pci_soft_state = PCI_SOFT_STATE_OPEN; 1126 } 1127 mutex_exit(&pci_p->pci_mutex); 1128 1129 return (0); 1130 } 1131 1132 static int 1133 npe_close(dev_t dev, int flags, int otyp, cred_t *credp) 1134 { 1135 minor_t minor = getminor(dev); 1136 int instance = PCI_MINOR_NUM_TO_INSTANCE(minor); 1137 pci_state_t *pci_p = ddi_get_soft_state(npe_statep, instance); 1138 int rv; 1139 1140 if (pci_p == NULL) 1141 return (ENXIO); 1142 1143 mutex_enter(&pci_p->pci_mutex); 1144 1145 switch (PCI_MINOR_NUM_TO_PCI_DEVNUM(minor)) { 1146 case PCI_TOOL_REG_MINOR_NUM: 1147 case PCI_TOOL_INTR_MINOR_NUM: 1148 break; 1149 default: 1150 /* Handle devctl ioctls */ 1151 rv = pcie_close(pci_p->pci_dip, dev, flags, otyp, credp); 1152 mutex_exit(&pci_p->pci_mutex); 1153 return (rv); 1154 } 1155 1156 /* Handle pcitool ioctls */ 1157 pci_p->pci_soft_state = PCI_SOFT_STATE_CLOSED; 1158 mutex_exit(&pci_p->pci_mutex); 1159 return (0); 1160 } 1161 1162 static int 1163 npe_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp) 1164 { 1165 minor_t minor = getminor(dev); 1166 int instance = PCI_MINOR_NUM_TO_INSTANCE(minor); 1167 pci_state_t *pci_p = ddi_get_soft_state(npe_statep, instance); 1168 int ret = ENOTTY; 1169 1170 if (pci_p == NULL) 1171 return (ENXIO); 1172 1173 switch (PCI_MINOR_NUM_TO_PCI_DEVNUM(minor)) { 1174 case PCI_TOOL_REG_MINOR_NUM: 1175 case PCI_TOOL_INTR_MINOR_NUM: 1176 /* To handle pcitool related ioctls */ 1177 ret = pci_common_ioctl(pci_p->pci_dip, dev, cmd, arg, mode, 1178 credp, rvalp); 1179 break; 1180 default: 1181 /* To handle devctl and hotplug related ioctls */ 1182 ret = pcie_ioctl(pci_p->pci_dip, dev, cmd, arg, mode, credp, 1183 rvalp); 1184 break; 1185 } 1186 1187 return (ret); 1188 } 1189 1190 /*ARGSUSED*/ 1191 static int 1192 npe_fm_init(dev_info_t *dip, dev_info_t *tdip, int cap, 1193 ddi_iblock_cookie_t *ibc) 1194 { 1195 pci_state_t *pcip = ddi_get_soft_state(npe_statep, 1196 ddi_get_instance(dip)); 1197 1198 ASSERT(ibc != NULL); 1199 *ibc = pcip->pci_fm_ibc; 1200 1201 return (pcip->pci_fmcap); 1202 } 1203 1204 static int 1205 npe_bus_get_eventcookie(dev_info_t *dip, dev_info_t *rdip, char *eventname, 1206 ddi_eventcookie_t *cookiep) 1207 { 1208 pci_state_t *pcip = ddi_get_soft_state(npe_statep, 1209 ddi_get_instance(dip)); 1210 1211 return (ndi_event_retrieve_cookie(pcip->pci_ndi_event_hdl, rdip, 1212 eventname, cookiep, NDI_EVENT_NOPASS)); 1213 } 1214 1215 static int 1216 npe_bus_add_eventcall(dev_info_t *dip, dev_info_t *rdip, 1217 ddi_eventcookie_t cookie, void (*callback)(dev_info_t *dip, 1218 ddi_eventcookie_t cookie, void *arg, void *bus_impldata), 1219 void *arg, ddi_callback_id_t *cb_id) 1220 { 1221 pci_state_t *pcip = ddi_get_soft_state(npe_statep, 1222 ddi_get_instance(dip)); 1223 1224 return (ndi_event_add_callback(pcip->pci_ndi_event_hdl, rdip, cookie, 1225 callback, arg, NDI_SLEEP, cb_id)); 1226 } 1227 1228 static int 1229 npe_bus_remove_eventcall(dev_info_t *dip, ddi_callback_id_t cb_id) 1230 { 1231 pci_state_t *pcip = ddi_get_soft_state(npe_statep, 1232 ddi_get_instance(dip)); 1233 return (ndi_event_remove_callback(pcip->pci_ndi_event_hdl, cb_id)); 1234 } 1235 1236 static int 1237 npe_bus_post_event(dev_info_t *dip, dev_info_t *rdip, 1238 ddi_eventcookie_t cookie, void *impl_data) 1239 { 1240 pci_state_t *pcip = ddi_get_soft_state(npe_statep, 1241 ddi_get_instance(dip)); 1242 return (ndi_event_do_callback(pcip->pci_ndi_event_hdl, rdip, cookie, 1243 impl_data)); 1244 1245 } 1246 1247 /*ARGSUSED*/ 1248 static int 1249 npe_fm_callback(dev_info_t *dip, ddi_fm_error_t *derr, const void *no_used) 1250 { 1251 /* 1252 * On current x86 systems, npe's callback does not get called for failed 1253 * loads. If in the future this feature is used, the fault PA should be 1254 * logged in the derr->fme_bus_specific field. The appropriate PCIe 1255 * error handling code should be called and needs to be coordinated with 1256 * safe access handling. 1257 */ 1258 1259 return (DDI_FM_OK); 1260 } 1261