1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 #include <sys/types.h> 26 #include <sys/kmem.h> 27 #include <sys/conf.h> 28 #include <sys/ddi.h> 29 #include <sys/sunddi.h> 30 #include <sys/sunndi.h> 31 #include <sys/fm/protocol.h> 32 #include <sys/fm/util.h> 33 #include <sys/modctl.h> 34 #include <sys/disp.h> 35 #include <sys/stat.h> 36 #include <sys/ddi_impldefs.h> 37 #include <sys/vmem.h> 38 #include <sys/iommutsb.h> 39 #include <sys/cpuvar.h> 40 #include <sys/ivintr.h> 41 #include <sys/byteorder.h> 42 #include <sys/spl.h> 43 #include <px_obj.h> 44 #include <sys/pcie_pwr.h> 45 #include "px_tools_var.h" 46 #include <px_regs.h> 47 #include <px_csr.h> 48 #include <sys/machsystm.h> 49 #include "px_lib4u.h" 50 #include "px_err.h" 51 #include "oberon_regs.h" 52 #include <sys/hotplug/pci/pcie_hp.h> 53 54 #pragma weak jbus_stst_order 55 56 extern void jbus_stst_order(); 57 58 ulong_t px_mmu_dvma_end = 0xfffffffful; 59 uint_t px_ranges_phi_mask = 0xfffffffful; 60 uint64_t *px_oberon_ubc_scratch_regs; 61 uint64_t px_paddr_mask; 62 63 static int px_goto_l23ready(px_t *px_p); 64 static int px_goto_l0(px_t *px_p); 65 static int px_pre_pwron_check(px_t *px_p); 66 static uint32_t px_identity_init(px_t *px_p); 67 static boolean_t px_cpr_callb(void *arg, int code); 68 static uint_t px_cb_intr(caddr_t arg); 69 70 /* 71 * ACKNAK Latency Threshold Table. 72 * See Fire PRM 2.0 section 1.2.12.2, table 1-17. 73 */ 74 int px_acknak_timer_table[LINK_MAX_PKT_ARR_SIZE][LINK_WIDTH_ARR_SIZE] = { 75 {0xED, 0x49, 0x43, 0x30}, 76 {0x1A0, 0x76, 0x6B, 0x48}, 77 {0x22F, 0x9A, 0x56, 0x56}, 78 {0x42F, 0x11A, 0x96, 0x96}, 79 {0x82F, 0x21A, 0x116, 0x116}, 80 {0x102F, 0x41A, 0x216, 0x216} 81 }; 82 83 /* 84 * TxLink Replay Timer Latency Table 85 * See Fire PRM 2.0 sections 1.2.12.3, table 1-18. 86 */ 87 int px_replay_timer_table[LINK_MAX_PKT_ARR_SIZE][LINK_WIDTH_ARR_SIZE] = { 88 {0x379, 0x112, 0xFC, 0xB4}, 89 {0x618, 0x1BA, 0x192, 0x10E}, 90 {0x831, 0x242, 0x143, 0x143}, 91 {0xFB1, 0x422, 0x233, 0x233}, 92 {0x1EB0, 0x7E1, 0x412, 0x412}, 93 {0x3CB0, 0xF61, 0x7D2, 0x7D2} 94 }; 95 /* 96 * px_lib_map_registers 97 * 98 * This function is called from the attach routine to map the registers 99 * accessed by this driver. 100 * 101 * used by: px_attach() 102 * 103 * return value: DDI_FAILURE on failure 104 */ 105 int 106 px_lib_map_regs(pxu_t *pxu_p, dev_info_t *dip) 107 { 108 ddi_device_acc_attr_t attr; 109 px_reg_bank_t reg_bank = PX_REG_CSR; 110 111 DBG(DBG_ATTACH, dip, "px_lib_map_regs: pxu_p:0x%p, dip 0x%p\n", 112 pxu_p, dip); 113 114 attr.devacc_attr_version = DDI_DEVICE_ATTR_V0; 115 attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC; 116 attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 117 118 /* 119 * PCI CSR Base 120 */ 121 if (ddi_regs_map_setup(dip, reg_bank, &pxu_p->px_address[reg_bank], 122 0, 0, &attr, &pxu_p->px_ac[reg_bank]) != DDI_SUCCESS) { 123 goto fail; 124 } 125 126 reg_bank++; 127 128 /* 129 * XBUS CSR Base 130 */ 131 if (ddi_regs_map_setup(dip, reg_bank, &pxu_p->px_address[reg_bank], 132 0, 0, &attr, &pxu_p->px_ac[reg_bank]) != DDI_SUCCESS) { 133 goto fail; 134 } 135 136 pxu_p->px_address[reg_bank] -= FIRE_CONTROL_STATUS; 137 138 done: 139 for (; reg_bank >= PX_REG_CSR; reg_bank--) { 140 DBG(DBG_ATTACH, dip, "reg_bank 0x%x address 0x%p\n", 141 reg_bank, pxu_p->px_address[reg_bank]); 142 } 143 144 return (DDI_SUCCESS); 145 146 fail: 147 cmn_err(CE_WARN, "%s%d: unable to map reg entry %d\n", 148 ddi_driver_name(dip), ddi_get_instance(dip), reg_bank); 149 150 for (reg_bank--; reg_bank >= PX_REG_CSR; reg_bank--) { 151 pxu_p->px_address[reg_bank] = NULL; 152 ddi_regs_map_free(&pxu_p->px_ac[reg_bank]); 153 } 154 155 return (DDI_FAILURE); 156 } 157 158 /* 159 * px_lib_unmap_regs: 160 * 161 * This routine unmaps the registers mapped by map_px_registers. 162 * 163 * used by: px_detach(), and error conditions in px_attach() 164 * 165 * return value: none 166 */ 167 void 168 px_lib_unmap_regs(pxu_t *pxu_p) 169 { 170 int i; 171 172 for (i = 0; i < PX_REG_MAX; i++) { 173 if (pxu_p->px_ac[i]) 174 ddi_regs_map_free(&pxu_p->px_ac[i]); 175 } 176 } 177 178 int 179 px_lib_dev_init(dev_info_t *dip, devhandle_t *dev_hdl) 180 { 181 182 caddr_t xbc_csr_base, csr_base; 183 px_dvma_range_prop_t px_dvma_range; 184 pxu_t *pxu_p; 185 uint8_t chip_mask; 186 px_t *px_p = DIP_TO_STATE(dip); 187 px_chip_type_t chip_type = px_identity_init(px_p); 188 189 DBG(DBG_ATTACH, dip, "px_lib_dev_init: dip 0x%p", dip); 190 191 if (chip_type == PX_CHIP_UNIDENTIFIED) { 192 cmn_err(CE_WARN, "%s%d: Unrecognized Hardware Version\n", 193 NAMEINST(dip)); 194 return (DDI_FAILURE); 195 } 196 197 chip_mask = BITMASK(chip_type); 198 px_paddr_mask = (chip_type == PX_CHIP_FIRE) ? MMU_FIRE_PADDR_MASK : 199 MMU_OBERON_PADDR_MASK; 200 201 /* 202 * Allocate platform specific structure and link it to 203 * the px state structure. 204 */ 205 pxu_p = kmem_zalloc(sizeof (pxu_t), KM_SLEEP); 206 pxu_p->chip_type = chip_type; 207 pxu_p->portid = ddi_getprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 208 "portid", -1); 209 210 /* Map in the registers */ 211 if (px_lib_map_regs(pxu_p, dip) == DDI_FAILURE) { 212 kmem_free(pxu_p, sizeof (pxu_t)); 213 214 return (DDI_FAILURE); 215 } 216 217 xbc_csr_base = (caddr_t)pxu_p->px_address[PX_REG_XBC]; 218 csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR]; 219 220 pxu_p->tsb_cookie = iommu_tsb_alloc(pxu_p->portid); 221 pxu_p->tsb_size = iommu_tsb_cookie_to_size(pxu_p->tsb_cookie); 222 pxu_p->tsb_vaddr = iommu_tsb_cookie_to_va(pxu_p->tsb_cookie); 223 224 pxu_p->tsb_paddr = va_to_pa(pxu_p->tsb_vaddr); 225 226 /* 227 * Create "virtual-dma" property to support child devices 228 * needing to know DVMA range. 229 */ 230 px_dvma_range.dvma_base = (uint32_t)px_mmu_dvma_end + 1 231 - ((pxu_p->tsb_size >> 3) << MMU_PAGE_SHIFT); 232 px_dvma_range.dvma_len = (uint32_t) 233 px_mmu_dvma_end - px_dvma_range.dvma_base + 1; 234 235 (void) ddi_prop_update_int_array(DDI_DEV_T_NONE, dip, 236 "virtual-dma", (int *)&px_dvma_range, 237 sizeof (px_dvma_range_prop_t) / sizeof (int)); 238 /* 239 * Initilize all fire hardware specific blocks. 240 */ 241 hvio_cb_init(xbc_csr_base, pxu_p); 242 hvio_ib_init(csr_base, pxu_p); 243 hvio_pec_init(csr_base, pxu_p); 244 hvio_mmu_init(csr_base, pxu_p); 245 246 px_p->px_plat_p = (void *)pxu_p; 247 248 /* 249 * Initialize all the interrupt handlers 250 */ 251 switch (PX_CHIP_TYPE(pxu_p)) { 252 case PX_CHIP_OBERON: 253 /* 254 * Oberon hotplug uses SPARE3 field in ILU Error Log Enable 255 * register to indicate the status of leaf reset, 256 * we need to preserve the value of this bit, and keep it in 257 * px_ilu_log_mask to reflect the state of the bit 258 */ 259 if (CSR_BR(csr_base, ILU_ERROR_LOG_ENABLE, SPARE3)) 260 px_ilu_log_mask |= (1ull << 261 ILU_ERROR_LOG_ENABLE_SPARE3); 262 else 263 px_ilu_log_mask &= ~(1ull << 264 ILU_ERROR_LOG_ENABLE_SPARE3); 265 266 px_err_reg_setup_pcie(chip_mask, csr_base, PX_ERR_ENABLE); 267 break; 268 269 case PX_CHIP_FIRE: 270 px_err_reg_setup_pcie(chip_mask, csr_base, PX_ERR_ENABLE); 271 break; 272 273 default: 274 cmn_err(CE_WARN, "%s%d: PX primary bus Unknown\n", 275 ddi_driver_name(dip), ddi_get_instance(dip)); 276 return (DDI_FAILURE); 277 } 278 279 /* Initilize device handle */ 280 *dev_hdl = (devhandle_t)csr_base; 281 282 DBG(DBG_ATTACH, dip, "px_lib_dev_init: dev_hdl 0x%llx\n", *dev_hdl); 283 284 /* Sun4u always support fixed interrupt */ 285 px_p->px_supp_intr_types |= DDI_INTR_TYPE_FIXED; 286 287 return (DDI_SUCCESS); 288 } 289 290 int 291 px_lib_dev_fini(dev_info_t *dip) 292 { 293 caddr_t csr_base; 294 uint8_t chip_mask; 295 px_t *px_p = DIP_TO_STATE(dip); 296 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 297 298 DBG(DBG_DETACH, dip, "px_lib_dev_fini: dip 0x%p\n", dip); 299 300 /* 301 * Deinitialize all the interrupt handlers 302 */ 303 switch (PX_CHIP_TYPE(pxu_p)) { 304 case PX_CHIP_OBERON: 305 case PX_CHIP_FIRE: 306 chip_mask = BITMASK(PX_CHIP_TYPE(pxu_p)); 307 csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR]; 308 px_err_reg_setup_pcie(chip_mask, csr_base, PX_ERR_DISABLE); 309 break; 310 311 default: 312 cmn_err(CE_WARN, "%s%d: PX primary bus Unknown\n", 313 ddi_driver_name(dip), ddi_get_instance(dip)); 314 return (DDI_FAILURE); 315 } 316 317 iommu_tsb_free(pxu_p->tsb_cookie); 318 319 px_lib_unmap_regs((pxu_t *)px_p->px_plat_p); 320 kmem_free(px_p->px_plat_p, sizeof (pxu_t)); 321 px_p->px_plat_p = NULL; 322 (void) ddi_prop_remove(DDI_DEV_T_NONE, dip, "virtual-dma"); 323 324 return (DDI_SUCCESS); 325 } 326 327 /*ARGSUSED*/ 328 int 329 px_lib_intr_devino_to_sysino(dev_info_t *dip, devino_t devino, 330 sysino_t *sysino) 331 { 332 px_t *px_p = DIP_TO_STATE(dip); 333 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 334 uint64_t ret; 335 336 DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: dip 0x%p " 337 "devino 0x%x\n", dip, devino); 338 339 if ((ret = hvio_intr_devino_to_sysino(DIP_TO_HANDLE(dip), 340 pxu_p, devino, sysino)) != H_EOK) { 341 DBG(DBG_LIB_INT, dip, 342 "hvio_intr_devino_to_sysino failed, ret 0x%lx\n", ret); 343 return (DDI_FAILURE); 344 } 345 346 DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: sysino 0x%llx\n", 347 *sysino); 348 349 return (DDI_SUCCESS); 350 } 351 352 /*ARGSUSED*/ 353 int 354 px_lib_intr_getvalid(dev_info_t *dip, sysino_t sysino, 355 intr_valid_state_t *intr_valid_state) 356 { 357 uint64_t ret; 358 359 DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: dip 0x%p sysino 0x%llx\n", 360 dip, sysino); 361 362 if ((ret = hvio_intr_getvalid(DIP_TO_HANDLE(dip), 363 sysino, intr_valid_state)) != H_EOK) { 364 DBG(DBG_LIB_INT, dip, "hvio_intr_getvalid failed, ret 0x%lx\n", 365 ret); 366 return (DDI_FAILURE); 367 } 368 369 DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: intr_valid_state 0x%x\n", 370 *intr_valid_state); 371 372 return (DDI_SUCCESS); 373 } 374 375 /*ARGSUSED*/ 376 int 377 px_lib_intr_setvalid(dev_info_t *dip, sysino_t sysino, 378 intr_valid_state_t intr_valid_state) 379 { 380 uint64_t ret; 381 382 DBG(DBG_LIB_INT, dip, "px_lib_intr_setvalid: dip 0x%p sysino 0x%llx " 383 "intr_valid_state 0x%x\n", dip, sysino, intr_valid_state); 384 385 if ((ret = hvio_intr_setvalid(DIP_TO_HANDLE(dip), 386 sysino, intr_valid_state)) != H_EOK) { 387 DBG(DBG_LIB_INT, dip, "hvio_intr_setvalid failed, ret 0x%lx\n", 388 ret); 389 return (DDI_FAILURE); 390 } 391 392 return (DDI_SUCCESS); 393 } 394 395 /*ARGSUSED*/ 396 int 397 px_lib_intr_getstate(dev_info_t *dip, sysino_t sysino, 398 intr_state_t *intr_state) 399 { 400 uint64_t ret; 401 402 DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: dip 0x%p sysino 0x%llx\n", 403 dip, sysino); 404 405 if ((ret = hvio_intr_getstate(DIP_TO_HANDLE(dip), 406 sysino, intr_state)) != H_EOK) { 407 DBG(DBG_LIB_INT, dip, "hvio_intr_getstate failed, ret 0x%lx\n", 408 ret); 409 return (DDI_FAILURE); 410 } 411 412 DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: intr_state 0x%x\n", 413 *intr_state); 414 415 return (DDI_SUCCESS); 416 } 417 418 /*ARGSUSED*/ 419 int 420 px_lib_intr_setstate(dev_info_t *dip, sysino_t sysino, 421 intr_state_t intr_state) 422 { 423 uint64_t ret; 424 425 DBG(DBG_LIB_INT, dip, "px_lib_intr_setstate: dip 0x%p sysino 0x%llx " 426 "intr_state 0x%x\n", dip, sysino, intr_state); 427 428 if ((ret = hvio_intr_setstate(DIP_TO_HANDLE(dip), 429 sysino, intr_state)) != H_EOK) { 430 DBG(DBG_LIB_INT, dip, "hvio_intr_setstate failed, ret 0x%lx\n", 431 ret); 432 return (DDI_FAILURE); 433 } 434 435 return (DDI_SUCCESS); 436 } 437 438 /*ARGSUSED*/ 439 int 440 px_lib_intr_gettarget(dev_info_t *dip, sysino_t sysino, cpuid_t *cpuid) 441 { 442 px_t *px_p = DIP_TO_STATE(dip); 443 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 444 uint64_t ret; 445 446 DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: dip 0x%p sysino 0x%llx\n", 447 dip, sysino); 448 449 if ((ret = hvio_intr_gettarget(DIP_TO_HANDLE(dip), pxu_p, 450 sysino, cpuid)) != H_EOK) { 451 DBG(DBG_LIB_INT, dip, "hvio_intr_gettarget failed, ret 0x%lx\n", 452 ret); 453 return (DDI_FAILURE); 454 } 455 456 DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: cpuid 0x%x\n", cpuid); 457 458 return (DDI_SUCCESS); 459 } 460 461 /*ARGSUSED*/ 462 int 463 px_lib_intr_settarget(dev_info_t *dip, sysino_t sysino, cpuid_t cpuid) 464 { 465 px_t *px_p = DIP_TO_STATE(dip); 466 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 467 uint64_t ret; 468 469 DBG(DBG_LIB_INT, dip, "px_lib_intr_settarget: dip 0x%p sysino 0x%llx " 470 "cpuid 0x%x\n", dip, sysino, cpuid); 471 472 if ((ret = hvio_intr_settarget(DIP_TO_HANDLE(dip), pxu_p, 473 sysino, cpuid)) != H_EOK) { 474 DBG(DBG_LIB_INT, dip, "hvio_intr_settarget failed, ret 0x%lx\n", 475 ret); 476 return (DDI_FAILURE); 477 } 478 479 return (DDI_SUCCESS); 480 } 481 482 /*ARGSUSED*/ 483 int 484 px_lib_intr_reset(dev_info_t *dip) 485 { 486 devino_t ino; 487 sysino_t sysino; 488 489 DBG(DBG_LIB_INT, dip, "px_lib_intr_reset: dip 0x%p\n", dip); 490 491 /* Reset all Interrupts */ 492 for (ino = 0; ino < INTERRUPT_MAPPING_ENTRIES; ino++) { 493 if (px_lib_intr_devino_to_sysino(dip, ino, 494 &sysino) != DDI_SUCCESS) 495 return (BF_FATAL); 496 497 if (px_lib_intr_setstate(dip, sysino, 498 INTR_IDLE_STATE) != DDI_SUCCESS) 499 return (BF_FATAL); 500 } 501 502 return (BF_NONE); 503 } 504 505 /*ARGSUSED*/ 506 int 507 px_lib_iommu_map(dev_info_t *dip, tsbid_t tsbid, pages_t pages, 508 io_attributes_t attr, void *addr, size_t pfn_index, int flags) 509 { 510 px_t *px_p = DIP_TO_STATE(dip); 511 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 512 uint64_t ret; 513 514 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: dip 0x%p tsbid 0x%llx " 515 "pages 0x%x attr 0x%llx addr 0x%p pfn_index 0x%llx flags 0x%x\n", 516 dip, tsbid, pages, attr, addr, pfn_index, flags); 517 518 if ((ret = hvio_iommu_map(px_p->px_dev_hdl, pxu_p, tsbid, pages, 519 attr, addr, pfn_index, flags)) != H_EOK) { 520 DBG(DBG_LIB_DMA, dip, 521 "px_lib_iommu_map failed, ret 0x%lx\n", ret); 522 return (DDI_FAILURE); 523 } 524 525 return (DDI_SUCCESS); 526 } 527 528 /*ARGSUSED*/ 529 int 530 px_lib_iommu_demap(dev_info_t *dip, tsbid_t tsbid, pages_t pages) 531 { 532 px_t *px_p = DIP_TO_STATE(dip); 533 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 534 uint64_t ret; 535 536 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: dip 0x%p tsbid 0x%llx " 537 "pages 0x%x\n", dip, tsbid, pages); 538 539 if ((ret = hvio_iommu_demap(px_p->px_dev_hdl, pxu_p, tsbid, pages)) 540 != H_EOK) { 541 DBG(DBG_LIB_DMA, dip, 542 "px_lib_iommu_demap failed, ret 0x%lx\n", ret); 543 544 return (DDI_FAILURE); 545 } 546 547 return (DDI_SUCCESS); 548 } 549 550 /*ARGSUSED*/ 551 int 552 px_lib_iommu_getmap(dev_info_t *dip, tsbid_t tsbid, io_attributes_t *attr_p, 553 r_addr_t *r_addr_p) 554 { 555 px_t *px_p = DIP_TO_STATE(dip); 556 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 557 uint64_t ret; 558 559 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: dip 0x%p tsbid 0x%llx\n", 560 dip, tsbid); 561 562 if ((ret = hvio_iommu_getmap(DIP_TO_HANDLE(dip), pxu_p, tsbid, 563 attr_p, r_addr_p)) != H_EOK) { 564 DBG(DBG_LIB_DMA, dip, 565 "hvio_iommu_getmap failed, ret 0x%lx\n", ret); 566 567 return ((ret == H_ENOMAP) ? DDI_DMA_NOMAPPING:DDI_FAILURE); 568 } 569 570 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: attr 0x%llx " 571 "r_addr 0x%llx\n", *attr_p, *r_addr_p); 572 573 return (DDI_SUCCESS); 574 } 575 576 int 577 px_lib_iommu_detach(px_t *px_p) 578 { 579 /* 580 * Deallocate DVMA addr space that was reserved for OBP TTE's 581 * during Attach. 582 */ 583 hvio_obptsb_detach(px_p); 584 585 return (DDI_SUCCESS); 586 } 587 588 /* 589 * Checks dma attributes against system bypass ranges 590 * The bypass range is determined by the hardware. Return them so the 591 * common code can do generic checking against them. 592 */ 593 /*ARGSUSED*/ 594 int 595 px_lib_dma_bypass_rngchk(dev_info_t *dip, ddi_dma_attr_t *attr_p, 596 uint64_t *lo_p, uint64_t *hi_p) 597 { 598 px_t *px_p = DIP_TO_STATE(dip); 599 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 600 601 *lo_p = hvio_get_bypass_base(pxu_p); 602 *hi_p = hvio_get_bypass_end(pxu_p); 603 604 return (DDI_SUCCESS); 605 } 606 607 608 /*ARGSUSED*/ 609 int 610 px_lib_iommu_getbypass(dev_info_t *dip, r_addr_t ra, io_attributes_t attr, 611 io_addr_t *io_addr_p) 612 { 613 uint64_t ret; 614 px_t *px_p = DIP_TO_STATE(dip); 615 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 616 617 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: dip 0x%p ra 0x%llx " 618 "attr 0x%llx\n", dip, ra, attr); 619 620 if ((ret = hvio_iommu_getbypass(DIP_TO_HANDLE(dip), pxu_p, ra, 621 attr, io_addr_p)) != H_EOK) { 622 DBG(DBG_LIB_DMA, dip, 623 "hvio_iommu_getbypass failed, ret 0x%lx\n", ret); 624 return (DDI_FAILURE); 625 } 626 627 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: io_addr 0x%llx\n", 628 *io_addr_p); 629 630 return (DDI_SUCCESS); 631 } 632 633 /* 634 * Returns any needed IO address bit(s) for relaxed ordering in IOMMU 635 * bypass mode. 636 */ 637 uint64_t 638 px_lib_ro_bypass(dev_info_t *dip, io_attributes_t attr, uint64_t ioaddr) 639 { 640 px_t *px_p = DIP_TO_STATE(dip); 641 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 642 643 if ((PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON) && (attr & PCI_MAP_ATTR_RO)) 644 return (MMU_OBERON_BYPASS_RO | ioaddr); 645 else 646 return (ioaddr); 647 } 648 649 /* 650 * bus dma sync entry point. 651 */ 652 /*ARGSUSED*/ 653 int 654 px_lib_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 655 off_t off, size_t len, uint_t cache_flags) 656 { 657 ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; 658 px_t *px_p = DIP_TO_STATE(dip); 659 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 660 661 DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: dip 0x%p rdip 0x%p " 662 "handle 0x%llx off 0x%x len 0x%x flags 0x%x\n", 663 dip, rdip, handle, off, len, cache_flags); 664 665 /* 666 * No flush needed for Oberon 667 */ 668 if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON) 669 return (DDI_SUCCESS); 670 671 /* 672 * jbus_stst_order is found only in certain cpu modules. 673 * Just return success if not present. 674 */ 675 if (&jbus_stst_order == NULL) 676 return (DDI_SUCCESS); 677 678 if (!(mp->dmai_flags & PX_DMAI_FLAGS_INUSE)) { 679 cmn_err(CE_WARN, "%s%d: Unbound dma handle %p.", 680 ddi_driver_name(rdip), ddi_get_instance(rdip), (void *)mp); 681 682 return (DDI_FAILURE); 683 } 684 685 if (mp->dmai_flags & PX_DMAI_FLAGS_NOSYNC) 686 return (DDI_SUCCESS); 687 688 /* 689 * No flush needed when sending data from memory to device. 690 * Nothing to do to "sync" memory to what device would already see. 691 */ 692 if (!(mp->dmai_rflags & DDI_DMA_READ) || 693 ((cache_flags & PX_DMA_SYNC_DDI_FLAGS) == DDI_DMA_SYNC_FORDEV)) 694 return (DDI_SUCCESS); 695 696 /* 697 * Perform necessary cpu workaround to ensure jbus ordering. 698 * CPU's internal "invalidate FIFOs" are flushed. 699 */ 700 701 kpreempt_disable(); 702 jbus_stst_order(); 703 kpreempt_enable(); 704 return (DDI_SUCCESS); 705 } 706 707 /* 708 * MSIQ Functions: 709 */ 710 /*ARGSUSED*/ 711 int 712 px_lib_msiq_init(dev_info_t *dip) 713 { 714 px_t *px_p = DIP_TO_STATE(dip); 715 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 716 px_msiq_state_t *msiq_state_p = &px_p->px_ib_p->ib_msiq_state; 717 px_dvma_addr_t pg_index; 718 size_t q_sz = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t); 719 size_t size; 720 int i, ret; 721 722 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_init: dip 0x%p\n", dip); 723 724 /* must aligned on q_sz (happens to be !!! page) boundary */ 725 ASSERT(q_sz == 8 * 1024); 726 727 /* 728 * Map the EQ memory into the Fire MMU (has to be 512KB aligned) 729 * and then initialize the base address register. 730 * 731 * Allocate entries from Fire IOMMU so that the resulting address 732 * is properly aligned. Calculate the index of the first allocated 733 * entry. Note: The size of the mapping is assumed to be a multiple 734 * of the page size. 735 */ 736 size = msiq_state_p->msiq_cnt * q_sz; 737 738 msiq_state_p->msiq_buf_p = kmem_zalloc(size, KM_SLEEP); 739 740 for (i = 0; i < msiq_state_p->msiq_cnt; i++) 741 msiq_state_p->msiq_p[i].msiq_base_p = (msiqhead_t *) 742 ((caddr_t)msiq_state_p->msiq_buf_p + (i * q_sz)); 743 744 pxu_p->msiq_mapped_p = vmem_xalloc(px_p->px_mmu_p->mmu_dvma_map, 745 size, (512 * 1024), 0, 0, NULL, NULL, VM_NOSLEEP | VM_BESTFIT); 746 747 if (pxu_p->msiq_mapped_p == NULL) 748 return (DDI_FAILURE); 749 750 pg_index = MMU_PAGE_INDEX(px_p->px_mmu_p, 751 MMU_BTOP((ulong_t)pxu_p->msiq_mapped_p)); 752 753 if ((ret = px_lib_iommu_map(px_p->px_dip, PCI_TSBID(0, pg_index), 754 MMU_BTOP(size), PCI_MAP_ATTR_WRITE, msiq_state_p->msiq_buf_p, 755 0, MMU_MAP_BUF)) != DDI_SUCCESS) { 756 DBG(DBG_LIB_MSIQ, dip, 757 "px_lib_msiq_init: px_lib_iommu_map failed, " 758 "ret 0x%lx\n", ret); 759 760 (void) px_lib_msiq_fini(dip); 761 return (DDI_FAILURE); 762 } 763 764 if ((ret = hvio_msiq_init(DIP_TO_HANDLE(dip), 765 pxu_p)) != H_EOK) { 766 DBG(DBG_LIB_MSIQ, dip, 767 "hvio_msiq_init failed, ret 0x%lx\n", ret); 768 769 (void) px_lib_msiq_fini(dip); 770 return (DDI_FAILURE); 771 } 772 773 return (DDI_SUCCESS); 774 } 775 776 /*ARGSUSED*/ 777 int 778 px_lib_msiq_fini(dev_info_t *dip) 779 { 780 px_t *px_p = DIP_TO_STATE(dip); 781 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 782 px_msiq_state_t *msiq_state_p = &px_p->px_ib_p->ib_msiq_state; 783 px_dvma_addr_t pg_index; 784 size_t size; 785 786 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_fini: dip 0x%p\n", dip); 787 788 /* 789 * Unmap and free the EQ memory that had been mapped 790 * into the Fire IOMMU. 791 */ 792 size = msiq_state_p->msiq_cnt * 793 msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t); 794 795 pg_index = MMU_PAGE_INDEX(px_p->px_mmu_p, 796 MMU_BTOP((ulong_t)pxu_p->msiq_mapped_p)); 797 798 (void) px_lib_iommu_demap(px_p->px_dip, 799 PCI_TSBID(0, pg_index), MMU_BTOP(size)); 800 801 /* Free the entries from the Fire MMU */ 802 vmem_xfree(px_p->px_mmu_p->mmu_dvma_map, 803 (void *)pxu_p->msiq_mapped_p, size); 804 805 kmem_free(msiq_state_p->msiq_buf_p, msiq_state_p->msiq_cnt * 806 msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t)); 807 808 return (DDI_SUCCESS); 809 } 810 811 /*ARGSUSED*/ 812 int 813 px_lib_msiq_info(dev_info_t *dip, msiqid_t msiq_id, r_addr_t *ra_p, 814 uint_t *msiq_rec_cnt_p) 815 { 816 px_t *px_p = DIP_TO_STATE(dip); 817 px_msiq_state_t *msiq_state_p = &px_p->px_ib_p->ib_msiq_state; 818 size_t msiq_size; 819 820 DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: dip 0x%p msiq_id 0x%x\n", 821 dip, msiq_id); 822 823 msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t); 824 ra_p = (r_addr_t *)((caddr_t)msiq_state_p->msiq_buf_p + 825 (msiq_id * msiq_size)); 826 827 *msiq_rec_cnt_p = msiq_state_p->msiq_rec_cnt; 828 829 DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: ra_p 0x%p msiq_rec_cnt 0x%x\n", 830 ra_p, *msiq_rec_cnt_p); 831 832 return (DDI_SUCCESS); 833 } 834 835 /*ARGSUSED*/ 836 int 837 px_lib_msiq_getvalid(dev_info_t *dip, msiqid_t msiq_id, 838 pci_msiq_valid_state_t *msiq_valid_state) 839 { 840 uint64_t ret; 841 842 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: dip 0x%p msiq_id 0x%x\n", 843 dip, msiq_id); 844 845 if ((ret = hvio_msiq_getvalid(DIP_TO_HANDLE(dip), 846 msiq_id, msiq_valid_state)) != H_EOK) { 847 DBG(DBG_LIB_MSIQ, dip, 848 "hvio_msiq_getvalid failed, ret 0x%lx\n", ret); 849 return (DDI_FAILURE); 850 } 851 852 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: msiq_valid_state 0x%x\n", 853 *msiq_valid_state); 854 855 return (DDI_SUCCESS); 856 } 857 858 /*ARGSUSED*/ 859 int 860 px_lib_msiq_setvalid(dev_info_t *dip, msiqid_t msiq_id, 861 pci_msiq_valid_state_t msiq_valid_state) 862 { 863 uint64_t ret; 864 865 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setvalid: dip 0x%p msiq_id 0x%x " 866 "msiq_valid_state 0x%x\n", dip, msiq_id, msiq_valid_state); 867 868 if ((ret = hvio_msiq_setvalid(DIP_TO_HANDLE(dip), 869 msiq_id, msiq_valid_state)) != H_EOK) { 870 DBG(DBG_LIB_MSIQ, dip, 871 "hvio_msiq_setvalid failed, ret 0x%lx\n", ret); 872 return (DDI_FAILURE); 873 } 874 875 return (DDI_SUCCESS); 876 } 877 878 /*ARGSUSED*/ 879 int 880 px_lib_msiq_getstate(dev_info_t *dip, msiqid_t msiq_id, 881 pci_msiq_state_t *msiq_state) 882 { 883 uint64_t ret; 884 885 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: dip 0x%p msiq_id 0x%x\n", 886 dip, msiq_id); 887 888 if ((ret = hvio_msiq_getstate(DIP_TO_HANDLE(dip), 889 msiq_id, msiq_state)) != H_EOK) { 890 DBG(DBG_LIB_MSIQ, dip, 891 "hvio_msiq_getstate failed, ret 0x%lx\n", ret); 892 return (DDI_FAILURE); 893 } 894 895 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: msiq_state 0x%x\n", 896 *msiq_state); 897 898 return (DDI_SUCCESS); 899 } 900 901 /*ARGSUSED*/ 902 int 903 px_lib_msiq_setstate(dev_info_t *dip, msiqid_t msiq_id, 904 pci_msiq_state_t msiq_state) 905 { 906 uint64_t ret; 907 908 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setstate: dip 0x%p msiq_id 0x%x " 909 "msiq_state 0x%x\n", dip, msiq_id, msiq_state); 910 911 if ((ret = hvio_msiq_setstate(DIP_TO_HANDLE(dip), 912 msiq_id, msiq_state)) != H_EOK) { 913 DBG(DBG_LIB_MSIQ, dip, 914 "hvio_msiq_setstate failed, ret 0x%lx\n", ret); 915 return (DDI_FAILURE); 916 } 917 918 return (DDI_SUCCESS); 919 } 920 921 /*ARGSUSED*/ 922 int 923 px_lib_msiq_gethead(dev_info_t *dip, msiqid_t msiq_id, 924 msiqhead_t *msiq_head) 925 { 926 uint64_t ret; 927 928 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: dip 0x%p msiq_id 0x%x\n", 929 dip, msiq_id); 930 931 if ((ret = hvio_msiq_gethead(DIP_TO_HANDLE(dip), 932 msiq_id, msiq_head)) != H_EOK) { 933 DBG(DBG_LIB_MSIQ, dip, 934 "hvio_msiq_gethead failed, ret 0x%lx\n", ret); 935 return (DDI_FAILURE); 936 } 937 938 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: msiq_head 0x%x\n", 939 *msiq_head); 940 941 return (DDI_SUCCESS); 942 } 943 944 /*ARGSUSED*/ 945 int 946 px_lib_msiq_sethead(dev_info_t *dip, msiqid_t msiq_id, 947 msiqhead_t msiq_head) 948 { 949 uint64_t ret; 950 951 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_sethead: dip 0x%p msiq_id 0x%x " 952 "msiq_head 0x%x\n", dip, msiq_id, msiq_head); 953 954 if ((ret = hvio_msiq_sethead(DIP_TO_HANDLE(dip), 955 msiq_id, msiq_head)) != H_EOK) { 956 DBG(DBG_LIB_MSIQ, dip, 957 "hvio_msiq_sethead failed, ret 0x%lx\n", ret); 958 return (DDI_FAILURE); 959 } 960 961 return (DDI_SUCCESS); 962 } 963 964 /*ARGSUSED*/ 965 int 966 px_lib_msiq_gettail(dev_info_t *dip, msiqid_t msiq_id, 967 msiqtail_t *msiq_tail) 968 { 969 uint64_t ret; 970 971 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: dip 0x%p msiq_id 0x%x\n", 972 dip, msiq_id); 973 974 if ((ret = hvio_msiq_gettail(DIP_TO_HANDLE(dip), 975 msiq_id, msiq_tail)) != H_EOK) { 976 DBG(DBG_LIB_MSIQ, dip, 977 "hvio_msiq_gettail failed, ret 0x%lx\n", ret); 978 return (DDI_FAILURE); 979 } 980 981 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: msiq_tail 0x%x\n", 982 *msiq_tail); 983 984 return (DDI_SUCCESS); 985 } 986 987 /*ARGSUSED*/ 988 void 989 px_lib_get_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p, 990 msiq_rec_t *msiq_rec_p) 991 { 992 eq_rec_t *eq_rec_p = (eq_rec_t *)msiq_head_p; 993 994 DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: dip 0x%p eq_rec_p 0x%p\n", 995 dip, eq_rec_p); 996 997 if (!eq_rec_p->eq_rec_fmt_type) { 998 /* Set msiq_rec_type to zero */ 999 msiq_rec_p->msiq_rec_type = 0; 1000 1001 return; 1002 } 1003 1004 DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: EQ RECORD, " 1005 "eq_rec_rid 0x%llx eq_rec_fmt_type 0x%llx " 1006 "eq_rec_len 0x%llx eq_rec_addr0 0x%llx " 1007 "eq_rec_addr1 0x%llx eq_rec_data0 0x%llx " 1008 "eq_rec_data1 0x%llx\n", eq_rec_p->eq_rec_rid, 1009 eq_rec_p->eq_rec_fmt_type, eq_rec_p->eq_rec_len, 1010 eq_rec_p->eq_rec_addr0, eq_rec_p->eq_rec_addr1, 1011 eq_rec_p->eq_rec_data0, eq_rec_p->eq_rec_data1); 1012 1013 /* 1014 * Only upper 4 bits of eq_rec_fmt_type is used 1015 * to identify the EQ record type. 1016 */ 1017 switch (eq_rec_p->eq_rec_fmt_type >> 3) { 1018 case EQ_REC_MSI32: 1019 msiq_rec_p->msiq_rec_type = MSI32_REC; 1020 1021 msiq_rec_p->msiq_rec_data.msi.msi_data = 1022 eq_rec_p->eq_rec_data0; 1023 break; 1024 case EQ_REC_MSI64: 1025 msiq_rec_p->msiq_rec_type = MSI64_REC; 1026 1027 msiq_rec_p->msiq_rec_data.msi.msi_data = 1028 eq_rec_p->eq_rec_data0; 1029 break; 1030 case EQ_REC_MSG: 1031 msiq_rec_p->msiq_rec_type = MSG_REC; 1032 1033 msiq_rec_p->msiq_rec_data.msg.msg_route = 1034 eq_rec_p->eq_rec_fmt_type & 7; 1035 msiq_rec_p->msiq_rec_data.msg.msg_targ = eq_rec_p->eq_rec_rid; 1036 msiq_rec_p->msiq_rec_data.msg.msg_code = eq_rec_p->eq_rec_data0; 1037 break; 1038 default: 1039 cmn_err(CE_WARN, "%s%d: px_lib_get_msiq_rec: " 1040 "0x%x is an unknown EQ record type", 1041 ddi_driver_name(dip), ddi_get_instance(dip), 1042 (int)eq_rec_p->eq_rec_fmt_type); 1043 break; 1044 } 1045 1046 msiq_rec_p->msiq_rec_rid = eq_rec_p->eq_rec_rid; 1047 msiq_rec_p->msiq_rec_msi_addr = ((eq_rec_p->eq_rec_addr1 << 16) | 1048 (eq_rec_p->eq_rec_addr0 << 2)); 1049 } 1050 1051 /*ARGSUSED*/ 1052 void 1053 px_lib_clr_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p) 1054 { 1055 eq_rec_t *eq_rec_p = (eq_rec_t *)msiq_head_p; 1056 1057 DBG(DBG_LIB_MSIQ, dip, "px_lib_clr_msiq_rec: dip 0x%p eq_rec_p 0x%p\n", 1058 dip, eq_rec_p); 1059 1060 if (eq_rec_p->eq_rec_fmt_type) { 1061 /* Zero out eq_rec_fmt_type field */ 1062 eq_rec_p->eq_rec_fmt_type = 0; 1063 } 1064 } 1065 1066 /* 1067 * MSI Functions: 1068 */ 1069 /*ARGSUSED*/ 1070 int 1071 px_lib_msi_init(dev_info_t *dip) 1072 { 1073 px_t *px_p = DIP_TO_STATE(dip); 1074 px_msi_state_t *msi_state_p = &px_p->px_ib_p->ib_msi_state; 1075 uint64_t ret; 1076 1077 DBG(DBG_LIB_MSI, dip, "px_lib_msi_init: dip 0x%p\n", dip); 1078 1079 if ((ret = hvio_msi_init(DIP_TO_HANDLE(dip), 1080 msi_state_p->msi_addr32, msi_state_p->msi_addr64)) != H_EOK) { 1081 DBG(DBG_LIB_MSIQ, dip, "px_lib_msi_init failed, ret 0x%lx\n", 1082 ret); 1083 return (DDI_FAILURE); 1084 } 1085 1086 return (DDI_SUCCESS); 1087 } 1088 1089 /*ARGSUSED*/ 1090 int 1091 px_lib_msi_getmsiq(dev_info_t *dip, msinum_t msi_num, 1092 msiqid_t *msiq_id) 1093 { 1094 uint64_t ret; 1095 1096 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: dip 0x%p msi_num 0x%x\n", 1097 dip, msi_num); 1098 1099 if ((ret = hvio_msi_getmsiq(DIP_TO_HANDLE(dip), 1100 msi_num, msiq_id)) != H_EOK) { 1101 DBG(DBG_LIB_MSI, dip, 1102 "hvio_msi_getmsiq failed, ret 0x%lx\n", ret); 1103 return (DDI_FAILURE); 1104 } 1105 1106 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: msiq_id 0x%x\n", 1107 *msiq_id); 1108 1109 return (DDI_SUCCESS); 1110 } 1111 1112 /*ARGSUSED*/ 1113 int 1114 px_lib_msi_setmsiq(dev_info_t *dip, msinum_t msi_num, 1115 msiqid_t msiq_id, msi_type_t msitype) 1116 { 1117 uint64_t ret; 1118 1119 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setmsiq: dip 0x%p msi_num 0x%x " 1120 "msq_id 0x%x\n", dip, msi_num, msiq_id); 1121 1122 if ((ret = hvio_msi_setmsiq(DIP_TO_HANDLE(dip), 1123 msi_num, msiq_id)) != H_EOK) { 1124 DBG(DBG_LIB_MSI, dip, 1125 "hvio_msi_setmsiq failed, ret 0x%lx\n", ret); 1126 return (DDI_FAILURE); 1127 } 1128 1129 return (DDI_SUCCESS); 1130 } 1131 1132 /*ARGSUSED*/ 1133 int 1134 px_lib_msi_getvalid(dev_info_t *dip, msinum_t msi_num, 1135 pci_msi_valid_state_t *msi_valid_state) 1136 { 1137 uint64_t ret; 1138 1139 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: dip 0x%p msi_num 0x%x\n", 1140 dip, msi_num); 1141 1142 if ((ret = hvio_msi_getvalid(DIP_TO_HANDLE(dip), 1143 msi_num, msi_valid_state)) != H_EOK) { 1144 DBG(DBG_LIB_MSI, dip, 1145 "hvio_msi_getvalid failed, ret 0x%lx\n", ret); 1146 return (DDI_FAILURE); 1147 } 1148 1149 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: msiq_id 0x%x\n", 1150 *msi_valid_state); 1151 1152 return (DDI_SUCCESS); 1153 } 1154 1155 /*ARGSUSED*/ 1156 int 1157 px_lib_msi_setvalid(dev_info_t *dip, msinum_t msi_num, 1158 pci_msi_valid_state_t msi_valid_state) 1159 { 1160 uint64_t ret; 1161 1162 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setvalid: dip 0x%p msi_num 0x%x " 1163 "msi_valid_state 0x%x\n", dip, msi_num, msi_valid_state); 1164 1165 if ((ret = hvio_msi_setvalid(DIP_TO_HANDLE(dip), 1166 msi_num, msi_valid_state)) != H_EOK) { 1167 DBG(DBG_LIB_MSI, dip, 1168 "hvio_msi_setvalid failed, ret 0x%lx\n", ret); 1169 return (DDI_FAILURE); 1170 } 1171 1172 return (DDI_SUCCESS); 1173 } 1174 1175 /*ARGSUSED*/ 1176 int 1177 px_lib_msi_getstate(dev_info_t *dip, msinum_t msi_num, 1178 pci_msi_state_t *msi_state) 1179 { 1180 uint64_t ret; 1181 1182 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: dip 0x%p msi_num 0x%x\n", 1183 dip, msi_num); 1184 1185 if ((ret = hvio_msi_getstate(DIP_TO_HANDLE(dip), 1186 msi_num, msi_state)) != H_EOK) { 1187 DBG(DBG_LIB_MSI, dip, 1188 "hvio_msi_getstate failed, ret 0x%lx\n", ret); 1189 return (DDI_FAILURE); 1190 } 1191 1192 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: msi_state 0x%x\n", 1193 *msi_state); 1194 1195 return (DDI_SUCCESS); 1196 } 1197 1198 /*ARGSUSED*/ 1199 int 1200 px_lib_msi_setstate(dev_info_t *dip, msinum_t msi_num, 1201 pci_msi_state_t msi_state) 1202 { 1203 uint64_t ret; 1204 1205 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setstate: dip 0x%p msi_num 0x%x " 1206 "msi_state 0x%x\n", dip, msi_num, msi_state); 1207 1208 if ((ret = hvio_msi_setstate(DIP_TO_HANDLE(dip), 1209 msi_num, msi_state)) != H_EOK) { 1210 DBG(DBG_LIB_MSI, dip, 1211 "hvio_msi_setstate failed, ret 0x%lx\n", ret); 1212 return (DDI_FAILURE); 1213 } 1214 1215 return (DDI_SUCCESS); 1216 } 1217 1218 /* 1219 * MSG Functions: 1220 */ 1221 /*ARGSUSED*/ 1222 int 1223 px_lib_msg_getmsiq(dev_info_t *dip, pcie_msg_type_t msg_type, 1224 msiqid_t *msiq_id) 1225 { 1226 uint64_t ret; 1227 1228 DBG(DBG_LIB_MSG, dip, "px_lib_msg_getmsiq: dip 0x%p msg_type 0x%x\n", 1229 dip, msg_type); 1230 1231 if ((ret = hvio_msg_getmsiq(DIP_TO_HANDLE(dip), 1232 msg_type, msiq_id)) != H_EOK) { 1233 DBG(DBG_LIB_MSG, dip, 1234 "hvio_msg_getmsiq failed, ret 0x%lx\n", ret); 1235 return (DDI_FAILURE); 1236 } 1237 1238 DBG(DBG_LIB_MSI, dip, "px_lib_msg_getmsiq: msiq_id 0x%x\n", 1239 *msiq_id); 1240 1241 return (DDI_SUCCESS); 1242 } 1243 1244 /*ARGSUSED*/ 1245 int 1246 px_lib_msg_setmsiq(dev_info_t *dip, pcie_msg_type_t msg_type, 1247 msiqid_t msiq_id) 1248 { 1249 uint64_t ret; 1250 1251 DBG(DBG_LIB_MSG, dip, "px_lib_msi_setstate: dip 0x%p msg_type 0x%x " 1252 "msiq_id 0x%x\n", dip, msg_type, msiq_id); 1253 1254 if ((ret = hvio_msg_setmsiq(DIP_TO_HANDLE(dip), 1255 msg_type, msiq_id)) != H_EOK) { 1256 DBG(DBG_LIB_MSG, dip, 1257 "hvio_msg_setmsiq failed, ret 0x%lx\n", ret); 1258 return (DDI_FAILURE); 1259 } 1260 1261 return (DDI_SUCCESS); 1262 } 1263 1264 /*ARGSUSED*/ 1265 int 1266 px_lib_msg_getvalid(dev_info_t *dip, pcie_msg_type_t msg_type, 1267 pcie_msg_valid_state_t *msg_valid_state) 1268 { 1269 uint64_t ret; 1270 1271 DBG(DBG_LIB_MSG, dip, "px_lib_msg_getvalid: dip 0x%p msg_type 0x%x\n", 1272 dip, msg_type); 1273 1274 if ((ret = hvio_msg_getvalid(DIP_TO_HANDLE(dip), msg_type, 1275 msg_valid_state)) != H_EOK) { 1276 DBG(DBG_LIB_MSG, dip, 1277 "hvio_msg_getvalid failed, ret 0x%lx\n", ret); 1278 return (DDI_FAILURE); 1279 } 1280 1281 DBG(DBG_LIB_MSI, dip, "px_lib_msg_getvalid: msg_valid_state 0x%x\n", 1282 *msg_valid_state); 1283 1284 return (DDI_SUCCESS); 1285 } 1286 1287 /*ARGSUSED*/ 1288 int 1289 px_lib_msg_setvalid(dev_info_t *dip, pcie_msg_type_t msg_type, 1290 pcie_msg_valid_state_t msg_valid_state) 1291 { 1292 uint64_t ret; 1293 1294 DBG(DBG_LIB_MSG, dip, "px_lib_msg_setvalid: dip 0x%p msg_type 0x%x " 1295 "msg_valid_state 0x%x\n", dip, msg_type, msg_valid_state); 1296 1297 if ((ret = hvio_msg_setvalid(DIP_TO_HANDLE(dip), msg_type, 1298 msg_valid_state)) != H_EOK) { 1299 DBG(DBG_LIB_MSG, dip, 1300 "hvio_msg_setvalid failed, ret 0x%lx\n", ret); 1301 return (DDI_FAILURE); 1302 } 1303 1304 return (DDI_SUCCESS); 1305 } 1306 1307 /*ARGSUSED*/ 1308 void 1309 px_panic_domain(px_t *px_p, pcie_req_id_t bdf) 1310 { 1311 } 1312 1313 /* 1314 * Suspend/Resume Functions: 1315 * Currently unsupported by hypervisor 1316 */ 1317 int 1318 px_lib_suspend(dev_info_t *dip) 1319 { 1320 px_t *px_p = DIP_TO_STATE(dip); 1321 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 1322 px_cb_t *cb_p = PX2CB(px_p); 1323 devhandle_t dev_hdl, xbus_dev_hdl; 1324 uint64_t ret = H_EOK; 1325 1326 DBG(DBG_DETACH, dip, "px_lib_suspend: dip 0x%p\n", dip); 1327 1328 dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_CSR]; 1329 xbus_dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_XBC]; 1330 1331 if ((ret = hvio_suspend(dev_hdl, pxu_p)) != H_EOK) 1332 goto fail; 1333 1334 if (--cb_p->attachcnt == 0) { 1335 ret = hvio_cb_suspend(xbus_dev_hdl, pxu_p); 1336 if (ret != H_EOK) 1337 cb_p->attachcnt++; 1338 } 1339 pxu_p->cpr_flag = PX_ENTERED_CPR; 1340 1341 fail: 1342 return ((ret != H_EOK) ? DDI_FAILURE: DDI_SUCCESS); 1343 } 1344 1345 void 1346 px_lib_resume(dev_info_t *dip) 1347 { 1348 px_t *px_p = DIP_TO_STATE(dip); 1349 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 1350 px_cb_t *cb_p = PX2CB(px_p); 1351 devhandle_t dev_hdl, xbus_dev_hdl; 1352 devino_t pec_ino = px_p->px_inos[PX_INTR_PEC]; 1353 devino_t xbc_ino = px_p->px_inos[PX_INTR_XBC]; 1354 1355 DBG(DBG_ATTACH, dip, "px_lib_resume: dip 0x%p\n", dip); 1356 1357 dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_CSR]; 1358 xbus_dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_XBC]; 1359 1360 if (++cb_p->attachcnt == 1) 1361 hvio_cb_resume(dev_hdl, xbus_dev_hdl, xbc_ino, pxu_p); 1362 1363 hvio_resume(dev_hdl, pec_ino, pxu_p); 1364 } 1365 1366 /* 1367 * Generate a unique Oberon UBC ID based on the Logicial System Board and 1368 * the IO Channel from the portid property field. 1369 */ 1370 static uint64_t 1371 oberon_get_ubc_id(dev_info_t *dip) 1372 { 1373 px_t *px_p = DIP_TO_STATE(dip); 1374 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 1375 uint64_t ubc_id; 1376 1377 /* 1378 * Generate a unique 6 bit UBC ID using the 2 IO_Channel#[1:0] bits and 1379 * the 4 LSB_ID[3:0] bits from the Oberon's portid property. 1380 */ 1381 ubc_id = (((pxu_p->portid >> OBERON_PORT_ID_IOC) & 1382 OBERON_PORT_ID_IOC_MASK) | (((pxu_p->portid >> 1383 OBERON_PORT_ID_LSB) & OBERON_PORT_ID_LSB_MASK) 1384 << OBERON_UBC_ID_LSB)); 1385 1386 return (ubc_id); 1387 } 1388 1389 /* 1390 * Oberon does not have a UBC scratch register, so alloc an array of scratch 1391 * registers when needed and use a unique UBC ID as an index. This code 1392 * can be simplified if we use a pre-allocated array. They are currently 1393 * being dynamically allocated because it's only needed by the Oberon. 1394 */ 1395 static void 1396 oberon_set_cb(dev_info_t *dip, uint64_t val) 1397 { 1398 uint64_t ubc_id; 1399 1400 if (px_oberon_ubc_scratch_regs == NULL) 1401 px_oberon_ubc_scratch_regs = 1402 (uint64_t *)kmem_zalloc(sizeof (uint64_t)* 1403 OBERON_UBC_ID_MAX, KM_SLEEP); 1404 1405 ubc_id = oberon_get_ubc_id(dip); 1406 1407 px_oberon_ubc_scratch_regs[ubc_id] = val; 1408 1409 /* 1410 * Check if any scratch registers are still in use. If all scratch 1411 * registers are currently set to zero, then deallocate the scratch 1412 * register array. 1413 */ 1414 for (ubc_id = 0; ubc_id < OBERON_UBC_ID_MAX; ubc_id++) { 1415 if (px_oberon_ubc_scratch_regs[ubc_id] != 0) 1416 return; 1417 } 1418 1419 /* 1420 * All scratch registers are set to zero so deallocate the scratch 1421 * register array and set the pointer to NULL. 1422 */ 1423 kmem_free(px_oberon_ubc_scratch_regs, 1424 (sizeof (uint64_t)*OBERON_UBC_ID_MAX)); 1425 1426 px_oberon_ubc_scratch_regs = NULL; 1427 } 1428 1429 /* 1430 * Oberon does not have a UBC scratch register, so use an allocated array of 1431 * scratch registers and use the unique UBC ID as an index into that array. 1432 */ 1433 static uint64_t 1434 oberon_get_cb(dev_info_t *dip) 1435 { 1436 uint64_t ubc_id; 1437 1438 if (px_oberon_ubc_scratch_regs == NULL) 1439 return (0); 1440 1441 ubc_id = oberon_get_ubc_id(dip); 1442 1443 return (px_oberon_ubc_scratch_regs[ubc_id]); 1444 } 1445 1446 /* 1447 * Misc Functions: 1448 * Currently unsupported by hypervisor 1449 */ 1450 static uint64_t 1451 px_get_cb(dev_info_t *dip) 1452 { 1453 px_t *px_p = DIP_TO_STATE(dip); 1454 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 1455 1456 /* 1457 * Oberon does not currently have Scratchpad registers. 1458 */ 1459 if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON) 1460 return (oberon_get_cb(dip)); 1461 1462 return (CSR_XR((caddr_t)pxu_p->px_address[PX_REG_XBC], JBUS_SCRATCH_1)); 1463 } 1464 1465 static void 1466 px_set_cb(dev_info_t *dip, uint64_t val) 1467 { 1468 px_t *px_p = DIP_TO_STATE(dip); 1469 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 1470 1471 /* 1472 * Oberon does not currently have Scratchpad registers. 1473 */ 1474 if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON) { 1475 oberon_set_cb(dip, val); 1476 return; 1477 } 1478 1479 CSR_XS((caddr_t)pxu_p->px_address[PX_REG_XBC], JBUS_SCRATCH_1, val); 1480 } 1481 1482 /*ARGSUSED*/ 1483 int 1484 px_lib_map_vconfig(dev_info_t *dip, 1485 ddi_map_req_t *mp, pci_config_offset_t off, 1486 pci_regspec_t *rp, caddr_t *addrp) 1487 { 1488 /* 1489 * No special config space access services in this layer. 1490 */ 1491 return (DDI_FAILURE); 1492 } 1493 1494 void 1495 px_lib_map_attr_check(ddi_map_req_t *mp) 1496 { 1497 ddi_acc_hdl_t *hp = mp->map_handlep; 1498 1499 /* fire does not accept byte masks from PIO store merge */ 1500 if (hp->ah_acc.devacc_attr_dataorder == DDI_STORECACHING_OK_ACC) 1501 hp->ah_acc.devacc_attr_dataorder = DDI_STRICTORDER_ACC; 1502 } 1503 1504 /* This function is called only by poke, caut put and pxtool poke. */ 1505 void 1506 px_lib_clr_errs(px_t *px_p, dev_info_t *rdip, uint64_t addr) 1507 { 1508 px_pec_t *pec_p = px_p->px_pec_p; 1509 dev_info_t *rpdip = px_p->px_dip; 1510 int rc_err, fab_err, i; 1511 int acctype = pec_p->pec_safeacc_type; 1512 ddi_fm_error_t derr; 1513 pci_ranges_t *ranges_p; 1514 int range_len; 1515 uint32_t addr_high, addr_low; 1516 pcie_req_id_t bdf = PCIE_INVALID_BDF; 1517 1518 /* Create the derr */ 1519 bzero(&derr, sizeof (ddi_fm_error_t)); 1520 derr.fme_version = DDI_FME_VERSION; 1521 derr.fme_ena = fm_ena_generate(0, FM_ENA_FMT1); 1522 derr.fme_flag = acctype; 1523 1524 if (acctype == DDI_FM_ERR_EXPECTED) { 1525 derr.fme_status = DDI_FM_NONFATAL; 1526 ndi_fm_acc_err_set(pec_p->pec_acc_hdl, &derr); 1527 } 1528 1529 if (px_fm_enter(px_p) != DDI_SUCCESS) 1530 return; 1531 1532 /* send ereport/handle/clear fire registers */ 1533 rc_err = px_err_cmn_intr(px_p, &derr, PX_LIB_CALL, PX_FM_BLOCK_ALL); 1534 1535 /* Figure out if this is a cfg or mem32 access */ 1536 addr_high = (uint32_t)(addr >> 32); 1537 addr_low = (uint32_t)addr; 1538 range_len = px_p->px_ranges_length / sizeof (pci_ranges_t); 1539 i = 0; 1540 for (ranges_p = px_p->px_ranges_p; i < range_len; i++, ranges_p++) { 1541 if (ranges_p->parent_high == addr_high) { 1542 switch (ranges_p->child_high & PCI_ADDR_MASK) { 1543 case PCI_ADDR_CONFIG: 1544 bdf = (pcie_req_id_t)(addr_low >> 12); 1545 addr_low = 0; 1546 break; 1547 case PCI_ADDR_MEM32: 1548 if (rdip) 1549 bdf = PCI_GET_BDF(rdip); 1550 else 1551 bdf = PCIE_INVALID_BDF; 1552 break; 1553 } 1554 break; 1555 } 1556 } 1557 1558 (void) px_rp_en_q(px_p, bdf, addr_low, 0); 1559 1560 /* 1561 * XXX - Current code scans the fabric for all px_tool accesses. 1562 * In future, do not scan fabric for px_tool access to IO Root Nexus 1563 */ 1564 fab_err = px_scan_fabric(px_p, rpdip, &derr); 1565 1566 px_err_panic(rc_err, PX_RC, fab_err, B_TRUE); 1567 px_fm_exit(px_p); 1568 px_err_panic(rc_err, PX_RC, fab_err, B_FALSE); 1569 } 1570 1571 #ifdef DEBUG 1572 int px_peekfault_cnt = 0; 1573 int px_pokefault_cnt = 0; 1574 #endif /* DEBUG */ 1575 1576 /*ARGSUSED*/ 1577 static int 1578 px_lib_do_poke(dev_info_t *dip, dev_info_t *rdip, 1579 peekpoke_ctlops_t *in_args) 1580 { 1581 px_t *px_p = DIP_TO_STATE(dip); 1582 px_pec_t *pec_p = px_p->px_pec_p; 1583 int err = DDI_SUCCESS; 1584 on_trap_data_t otd; 1585 1586 mutex_enter(&pec_p->pec_pokefault_mutex); 1587 pec_p->pec_ontrap_data = &otd; 1588 pec_p->pec_safeacc_type = DDI_FM_ERR_POKE; 1589 1590 /* Set up protected environment. */ 1591 if (!on_trap(&otd, OT_DATA_ACCESS)) { 1592 uintptr_t tramp = otd.ot_trampoline; 1593 1594 otd.ot_trampoline = (uintptr_t)&poke_fault; 1595 err = do_poke(in_args->size, (void *)in_args->dev_addr, 1596 (void *)in_args->host_addr); 1597 otd.ot_trampoline = tramp; 1598 } else 1599 err = DDI_FAILURE; 1600 1601 px_lib_clr_errs(px_p, rdip, in_args->dev_addr); 1602 1603 if (otd.ot_trap & OT_DATA_ACCESS) 1604 err = DDI_FAILURE; 1605 1606 /* Take down protected environment. */ 1607 no_trap(); 1608 1609 pec_p->pec_ontrap_data = NULL; 1610 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1611 mutex_exit(&pec_p->pec_pokefault_mutex); 1612 1613 #ifdef DEBUG 1614 if (err == DDI_FAILURE) 1615 px_pokefault_cnt++; 1616 #endif 1617 return (err); 1618 } 1619 1620 /*ARGSUSED*/ 1621 static int 1622 px_lib_do_caut_put(dev_info_t *dip, dev_info_t *rdip, 1623 peekpoke_ctlops_t *cautacc_ctlops_arg) 1624 { 1625 size_t size = cautacc_ctlops_arg->size; 1626 uintptr_t dev_addr = cautacc_ctlops_arg->dev_addr; 1627 uintptr_t host_addr = cautacc_ctlops_arg->host_addr; 1628 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)cautacc_ctlops_arg->handle; 1629 size_t repcount = cautacc_ctlops_arg->repcount; 1630 uint_t flags = cautacc_ctlops_arg->flags; 1631 1632 px_t *px_p = DIP_TO_STATE(dip); 1633 px_pec_t *pec_p = px_p->px_pec_p; 1634 int err = DDI_SUCCESS; 1635 1636 /* 1637 * Note that i_ndi_busop_access_enter ends up grabbing the pokefault 1638 * mutex. 1639 */ 1640 i_ndi_busop_access_enter(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp); 1641 1642 pec_p->pec_ontrap_data = (on_trap_data_t *)hp->ahi_err->err_ontrap; 1643 pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED; 1644 hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED; 1645 1646 if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) { 1647 for (; repcount; repcount--) { 1648 switch (size) { 1649 1650 case sizeof (uint8_t): 1651 i_ddi_put8(hp, (uint8_t *)dev_addr, 1652 *(uint8_t *)host_addr); 1653 break; 1654 1655 case sizeof (uint16_t): 1656 i_ddi_put16(hp, (uint16_t *)dev_addr, 1657 *(uint16_t *)host_addr); 1658 break; 1659 1660 case sizeof (uint32_t): 1661 i_ddi_put32(hp, (uint32_t *)dev_addr, 1662 *(uint32_t *)host_addr); 1663 break; 1664 1665 case sizeof (uint64_t): 1666 i_ddi_put64(hp, (uint64_t *)dev_addr, 1667 *(uint64_t *)host_addr); 1668 break; 1669 } 1670 1671 host_addr += size; 1672 1673 if (flags == DDI_DEV_AUTOINCR) 1674 dev_addr += size; 1675 1676 px_lib_clr_errs(px_p, rdip, dev_addr); 1677 1678 if (pec_p->pec_ontrap_data->ot_trap & OT_DATA_ACCESS) { 1679 err = DDI_FAILURE; 1680 #ifdef DEBUG 1681 px_pokefault_cnt++; 1682 #endif 1683 break; 1684 } 1685 } 1686 } 1687 1688 i_ddi_notrap((ddi_acc_handle_t)hp); 1689 pec_p->pec_ontrap_data = NULL; 1690 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1691 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp); 1692 hp->ahi_err->err_expected = DDI_FM_ERR_UNEXPECTED; 1693 1694 return (err); 1695 } 1696 1697 1698 int 1699 px_lib_ctlops_poke(dev_info_t *dip, dev_info_t *rdip, 1700 peekpoke_ctlops_t *in_args) 1701 { 1702 return (in_args->handle ? px_lib_do_caut_put(dip, rdip, in_args) : 1703 px_lib_do_poke(dip, rdip, in_args)); 1704 } 1705 1706 1707 /*ARGSUSED*/ 1708 static int 1709 px_lib_do_peek(dev_info_t *dip, peekpoke_ctlops_t *in_args) 1710 { 1711 px_t *px_p = DIP_TO_STATE(dip); 1712 px_pec_t *pec_p = px_p->px_pec_p; 1713 int err = DDI_SUCCESS; 1714 on_trap_data_t otd; 1715 1716 mutex_enter(&pec_p->pec_pokefault_mutex); 1717 if (px_fm_enter(px_p) != DDI_SUCCESS) { 1718 mutex_exit(&pec_p->pec_pokefault_mutex); 1719 return (DDI_FAILURE); 1720 } 1721 pec_p->pec_safeacc_type = DDI_FM_ERR_PEEK; 1722 px_fm_exit(px_p); 1723 1724 if (!on_trap(&otd, OT_DATA_ACCESS)) { 1725 uintptr_t tramp = otd.ot_trampoline; 1726 1727 otd.ot_trampoline = (uintptr_t)&peek_fault; 1728 err = do_peek(in_args->size, (void *)in_args->dev_addr, 1729 (void *)in_args->host_addr); 1730 otd.ot_trampoline = tramp; 1731 } else 1732 err = DDI_FAILURE; 1733 1734 no_trap(); 1735 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1736 mutex_exit(&pec_p->pec_pokefault_mutex); 1737 1738 #ifdef DEBUG 1739 if (err == DDI_FAILURE) 1740 px_peekfault_cnt++; 1741 #endif 1742 return (err); 1743 } 1744 1745 1746 static int 1747 px_lib_do_caut_get(dev_info_t *dip, peekpoke_ctlops_t *cautacc_ctlops_arg) 1748 { 1749 size_t size = cautacc_ctlops_arg->size; 1750 uintptr_t dev_addr = cautacc_ctlops_arg->dev_addr; 1751 uintptr_t host_addr = cautacc_ctlops_arg->host_addr; 1752 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)cautacc_ctlops_arg->handle; 1753 size_t repcount = cautacc_ctlops_arg->repcount; 1754 uint_t flags = cautacc_ctlops_arg->flags; 1755 1756 px_t *px_p = DIP_TO_STATE(dip); 1757 px_pec_t *pec_p = px_p->px_pec_p; 1758 int err = DDI_SUCCESS; 1759 1760 /* 1761 * Note that i_ndi_busop_access_enter ends up grabbing the pokefault 1762 * mutex. 1763 */ 1764 i_ndi_busop_access_enter(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp); 1765 1766 pec_p->pec_ontrap_data = (on_trap_data_t *)hp->ahi_err->err_ontrap; 1767 pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED; 1768 hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED; 1769 1770 if (repcount == 1) { 1771 if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) { 1772 i_ddi_caut_get(size, (void *)dev_addr, 1773 (void *)host_addr); 1774 } else { 1775 int i; 1776 uint8_t *ff_addr = (uint8_t *)host_addr; 1777 for (i = 0; i < size; i++) 1778 *ff_addr++ = 0xff; 1779 1780 err = DDI_FAILURE; 1781 #ifdef DEBUG 1782 px_peekfault_cnt++; 1783 #endif 1784 } 1785 } else { 1786 if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) { 1787 for (; repcount; repcount--) { 1788 i_ddi_caut_get(size, (void *)dev_addr, 1789 (void *)host_addr); 1790 1791 host_addr += size; 1792 1793 if (flags == DDI_DEV_AUTOINCR) 1794 dev_addr += size; 1795 } 1796 } else { 1797 err = DDI_FAILURE; 1798 #ifdef DEBUG 1799 px_peekfault_cnt++; 1800 #endif 1801 } 1802 } 1803 1804 i_ddi_notrap((ddi_acc_handle_t)hp); 1805 pec_p->pec_ontrap_data = NULL; 1806 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1807 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp); 1808 hp->ahi_err->err_expected = DDI_FM_ERR_UNEXPECTED; 1809 1810 return (err); 1811 } 1812 1813 /*ARGSUSED*/ 1814 int 1815 px_lib_ctlops_peek(dev_info_t *dip, dev_info_t *rdip, 1816 peekpoke_ctlops_t *in_args, void *result) 1817 { 1818 result = (void *)in_args->host_addr; 1819 return (in_args->handle ? px_lib_do_caut_get(dip, in_args) : 1820 px_lib_do_peek(dip, in_args)); 1821 } 1822 1823 /* 1824 * implements PPM interface 1825 */ 1826 int 1827 px_lib_pmctl(int cmd, px_t *px_p) 1828 { 1829 ASSERT((cmd & ~PPMREQ_MASK) == PPMREQ); 1830 switch (cmd) { 1831 case PPMREQ_PRE_PWR_OFF: 1832 /* 1833 * Currently there is no device power management for 1834 * the root complex (fire). When there is we need to make 1835 * sure that it is at full power before trying to send the 1836 * PME_Turn_Off message. 1837 */ 1838 DBG(DBG_PWR, px_p->px_dip, 1839 "ioctl: request to send PME_Turn_Off\n"); 1840 return (px_goto_l23ready(px_p)); 1841 1842 case PPMREQ_PRE_PWR_ON: 1843 DBG(DBG_PWR, px_p->px_dip, "ioctl: PRE_PWR_ON request\n"); 1844 return (px_pre_pwron_check(px_p)); 1845 1846 case PPMREQ_POST_PWR_ON: 1847 DBG(DBG_PWR, px_p->px_dip, "ioctl: POST_PWR_ON request\n"); 1848 return (px_goto_l0(px_p)); 1849 1850 default: 1851 return (DDI_FAILURE); 1852 } 1853 } 1854 1855 /* 1856 * sends PME_Turn_Off message to put the link in L2/L3 ready state. 1857 * called by px_ioctl. 1858 * returns DDI_SUCCESS or DDI_FAILURE 1859 * 1. Wait for link to be in L1 state (link status reg) 1860 * 2. write to PME_Turn_off reg to boradcast 1861 * 3. set timeout 1862 * 4. If timeout, return failure. 1863 * 5. If PM_TO_Ack, wait till link is in L2/L3 ready 1864 */ 1865 static int 1866 px_goto_l23ready(px_t *px_p) 1867 { 1868 pcie_pwr_t *pwr_p; 1869 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 1870 caddr_t csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR]; 1871 int ret = DDI_SUCCESS; 1872 clock_t end, timeleft; 1873 int mutex_held = 1; 1874 1875 /* If no PM info, return failure */ 1876 if (PCIE_PMINFO(px_p->px_dip) == NULL || 1877 (pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip)) == NULL) 1878 return (DDI_FAILURE); 1879 1880 mutex_enter(&pwr_p->pwr_lock); 1881 mutex_enter(&px_p->px_l23ready_lock); 1882 /* Clear the PME_To_ACK receieved flag */ 1883 px_p->px_pm_flags &= ~PX_PMETOACK_RECVD; 1884 /* 1885 * When P25 is the downstream device, after receiving 1886 * PME_To_ACK, fire will go to Detect state, which causes 1887 * the link down event. Inform FMA that this is expected. 1888 * In case of all other cards complaint with the pci express 1889 * spec, this will happen when the power is re-applied. FMA 1890 * code will clear this flag after one instance of LDN. Since 1891 * there will not be a LDN event for the spec compliant cards, 1892 * we need to clear the flag after receiving PME_To_ACK. 1893 */ 1894 px_p->px_pm_flags |= PX_LDN_EXPECTED; 1895 if (px_send_pme_turnoff(csr_base) != DDI_SUCCESS) { 1896 ret = DDI_FAILURE; 1897 goto l23ready_done; 1898 } 1899 px_p->px_pm_flags |= PX_PME_TURNOFF_PENDING; 1900 1901 end = ddi_get_lbolt() + drv_usectohz(px_pme_to_ack_timeout); 1902 while (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) { 1903 timeleft = cv_timedwait(&px_p->px_l23ready_cv, 1904 &px_p->px_l23ready_lock, end); 1905 /* 1906 * if cv_timedwait returns -1, it is either 1907 * 1) timed out or 1908 * 2) there was a pre-mature wakeup but by the time 1909 * cv_timedwait is called again end < lbolt i.e. 1910 * end is in the past. 1911 * 3) By the time we make first cv_timedwait call, 1912 * end < lbolt is true. 1913 */ 1914 if (timeleft == -1) 1915 break; 1916 } 1917 if (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) { 1918 /* 1919 * Either timedout or interrupt didn't get a 1920 * chance to grab the mutex and set the flag. 1921 * release the mutex and delay for sometime. 1922 * This will 1) give a chance for interrupt to 1923 * set the flag 2) creates a delay between two 1924 * consequetive requests. 1925 */ 1926 mutex_exit(&px_p->px_l23ready_lock); 1927 delay(drv_usectohz(50 * PX_MSEC_TO_USEC)); 1928 mutex_held = 0; 1929 if (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) { 1930 ret = DDI_FAILURE; 1931 DBG(DBG_PWR, px_p->px_dip, " Timed out while waiting" 1932 " for PME_TO_ACK\n"); 1933 } 1934 } 1935 px_p->px_pm_flags &= 1936 ~(PX_PME_TURNOFF_PENDING | PX_PMETOACK_RECVD | PX_LDN_EXPECTED); 1937 1938 l23ready_done: 1939 if (mutex_held) 1940 mutex_exit(&px_p->px_l23ready_lock); 1941 /* 1942 * Wait till link is in L1 idle, if sending PME_Turn_Off 1943 * was succesful. 1944 */ 1945 if (ret == DDI_SUCCESS) { 1946 if (px_link_wait4l1idle(csr_base) != DDI_SUCCESS) { 1947 DBG(DBG_PWR, px_p->px_dip, " Link is not at L1" 1948 " even though we received PME_To_ACK.\n"); 1949 /* 1950 * Workaround for hardware bug with P25. 1951 * Due to a hardware bug with P25, link state 1952 * will be Detect state rather than L1 after 1953 * link is transitioned to L23Ready state. Since 1954 * we don't know whether link is L23ready state 1955 * without Fire's state being L1_idle, we delay 1956 * here just to make sure that we wait till link 1957 * is transitioned to L23Ready state. 1958 */ 1959 delay(drv_usectohz(100 * PX_MSEC_TO_USEC)); 1960 } 1961 pwr_p->pwr_link_lvl = PM_LEVEL_L3; 1962 1963 } 1964 mutex_exit(&pwr_p->pwr_lock); 1965 return (ret); 1966 } 1967 1968 /* 1969 * Message interrupt handler intended to be shared for both 1970 * PME and PME_TO_ACK msg handling, currently only handles 1971 * PME_To_ACK message. 1972 */ 1973 uint_t 1974 px_pmeq_intr(caddr_t arg) 1975 { 1976 px_t *px_p = (px_t *)arg; 1977 1978 DBG(DBG_PWR, px_p->px_dip, " PME_To_ACK received \n"); 1979 mutex_enter(&px_p->px_l23ready_lock); 1980 cv_broadcast(&px_p->px_l23ready_cv); 1981 if (px_p->px_pm_flags & PX_PME_TURNOFF_PENDING) { 1982 px_p->px_pm_flags |= PX_PMETOACK_RECVD; 1983 } else { 1984 /* 1985 * This maybe the second ack received. If so then, 1986 * we should be receiving it during wait4L1 stage. 1987 */ 1988 px_p->px_pmetoack_ignored++; 1989 } 1990 mutex_exit(&px_p->px_l23ready_lock); 1991 return (DDI_INTR_CLAIMED); 1992 } 1993 1994 static int 1995 px_pre_pwron_check(px_t *px_p) 1996 { 1997 pcie_pwr_t *pwr_p; 1998 1999 /* If no PM info, return failure */ 2000 if (PCIE_PMINFO(px_p->px_dip) == NULL || 2001 (pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip)) == NULL) 2002 return (DDI_FAILURE); 2003 2004 /* 2005 * For the spec compliant downstream cards link down 2006 * is expected when the device is powered on. 2007 */ 2008 px_p->px_pm_flags |= PX_LDN_EXPECTED; 2009 return (pwr_p->pwr_link_lvl == PM_LEVEL_L3 ? DDI_SUCCESS : DDI_FAILURE); 2010 } 2011 2012 static int 2013 px_goto_l0(px_t *px_p) 2014 { 2015 pcie_pwr_t *pwr_p; 2016 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 2017 caddr_t csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR]; 2018 int ret = DDI_SUCCESS; 2019 uint64_t time_spent = 0; 2020 2021 /* If no PM info, return failure */ 2022 if (PCIE_PMINFO(px_p->px_dip) == NULL || 2023 (pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip)) == NULL) 2024 return (DDI_FAILURE); 2025 2026 mutex_enter(&pwr_p->pwr_lock); 2027 /* 2028 * The following link retrain activity will cause LDN and LUP event. 2029 * Receiving LDN prior to receiving LUP is expected, not an error in 2030 * this case. Receiving LUP indicates link is fully up to support 2031 * powering up down stream device, and of course any further LDN and 2032 * LUP outside this context will be error. 2033 */ 2034 px_p->px_lup_pending = 1; 2035 if (px_link_retrain(csr_base) != DDI_SUCCESS) { 2036 ret = DDI_FAILURE; 2037 goto l0_done; 2038 } 2039 2040 /* LUP event takes the order of 15ms amount of time to occur */ 2041 for (; px_p->px_lup_pending && (time_spent < px_lup_poll_to); 2042 time_spent += px_lup_poll_interval) 2043 drv_usecwait(px_lup_poll_interval); 2044 if (px_p->px_lup_pending) 2045 ret = DDI_FAILURE; 2046 l0_done: 2047 px_enable_detect_quiet(csr_base); 2048 if (ret == DDI_SUCCESS) 2049 pwr_p->pwr_link_lvl = PM_LEVEL_L0; 2050 mutex_exit(&pwr_p->pwr_lock); 2051 return (ret); 2052 } 2053 2054 /* 2055 * Extract the drivers binding name to identify which chip we're binding to. 2056 * Whenever a new bus bridge is created, the driver alias entry should be 2057 * added here to identify the device if needed. If a device isn't added, 2058 * the identity defaults to PX_CHIP_UNIDENTIFIED. 2059 */ 2060 static uint32_t 2061 px_identity_init(px_t *px_p) 2062 { 2063 dev_info_t *dip = px_p->px_dip; 2064 char *name = ddi_binding_name(dip); 2065 uint32_t revision = 0; 2066 2067 revision = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 2068 "module-revision#", 0); 2069 2070 /* Check for Fire driver binding name */ 2071 if (strcmp(name, "pciex108e,80f0") == 0) { 2072 DBG(DBG_ATTACH, dip, "px_identity_init: %s%d: " 2073 "(FIRE), module-revision %d\n", NAMEINST(dip), 2074 revision); 2075 2076 return ((revision >= FIRE_MOD_REV_20) ? 2077 PX_CHIP_FIRE : PX_CHIP_UNIDENTIFIED); 2078 } 2079 2080 /* Check for Oberon driver binding name */ 2081 if (strcmp(name, "pciex108e,80f8") == 0) { 2082 DBG(DBG_ATTACH, dip, "px_identity_init: %s%d: " 2083 "(OBERON), module-revision %d\n", NAMEINST(dip), 2084 revision); 2085 2086 return (PX_CHIP_OBERON); 2087 } 2088 2089 DBG(DBG_ATTACH, dip, "%s%d: Unknown PCI Express Host bridge %s %x\n", 2090 ddi_driver_name(dip), ddi_get_instance(dip), name, revision); 2091 2092 return (PX_CHIP_UNIDENTIFIED); 2093 } 2094 2095 int 2096 px_err_add_intr(px_fault_t *px_fault_p) 2097 { 2098 dev_info_t *dip = px_fault_p->px_fh_dip; 2099 px_t *px_p = DIP_TO_STATE(dip); 2100 2101 VERIFY(add_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL, 2102 (intrfunc)px_fault_p->px_err_func, (caddr_t)px_fault_p, 2103 NULL, NULL) == 0); 2104 2105 px_ib_intr_enable(px_p, intr_dist_cpuid(), px_fault_p->px_intr_ino); 2106 2107 return (DDI_SUCCESS); 2108 } 2109 2110 void 2111 px_err_rem_intr(px_fault_t *px_fault_p) 2112 { 2113 dev_info_t *dip = px_fault_p->px_fh_dip; 2114 px_t *px_p = DIP_TO_STATE(dip); 2115 2116 px_ib_intr_disable(px_p->px_ib_p, px_fault_p->px_intr_ino, 2117 IB_INTR_WAIT); 2118 2119 VERIFY(rem_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL) == 0); 2120 } 2121 2122 /* 2123 * px_cb_intr_redist() - sun4u only, CB interrupt redistribution 2124 */ 2125 void 2126 px_cb_intr_redist(void *arg) 2127 { 2128 px_cb_t *cb_p = (px_cb_t *)arg; 2129 px_cb_list_t *pxl; 2130 px_t *pxp = NULL; 2131 px_fault_t *f_p = NULL; 2132 uint32_t new_cpuid; 2133 intr_valid_state_t enabled = 0; 2134 2135 mutex_enter(&cb_p->cb_mutex); 2136 2137 pxl = cb_p->pxl; 2138 if (pxl == NULL) 2139 goto cb_done; 2140 2141 pxp = pxl->pxp; 2142 f_p = &pxp->px_cb_fault; 2143 for (; pxl && (f_p->px_fh_sysino != cb_p->sysino); ) { 2144 pxl = pxl->next; 2145 pxp = pxl->pxp; 2146 f_p = &pxp->px_cb_fault; 2147 } 2148 if (pxl == NULL) 2149 goto cb_done; 2150 2151 new_cpuid = intr_dist_cpuid(); 2152 if (new_cpuid == cb_p->cpuid) 2153 goto cb_done; 2154 2155 if ((px_lib_intr_getvalid(pxp->px_dip, f_p->px_fh_sysino, &enabled) 2156 != DDI_SUCCESS) || !enabled) { 2157 DBG(DBG_IB, pxp->px_dip, "px_cb_intr_redist: CB not enabled, " 2158 "sysino(0x%x)\n", f_p->px_fh_sysino); 2159 goto cb_done; 2160 } 2161 2162 PX_INTR_DISABLE(pxp->px_dip, f_p->px_fh_sysino); 2163 2164 cb_p->cpuid = new_cpuid; 2165 cb_p->sysino = f_p->px_fh_sysino; 2166 PX_INTR_ENABLE(pxp->px_dip, cb_p->sysino, cb_p->cpuid); 2167 2168 cb_done: 2169 mutex_exit(&cb_p->cb_mutex); 2170 } 2171 2172 /* 2173 * px_cb_add_intr() - Called from attach(9E) to create CB if not yet 2174 * created, to add CB interrupt vector always, but enable only once. 2175 */ 2176 int 2177 px_cb_add_intr(px_fault_t *fault_p) 2178 { 2179 px_t *px_p = DIP_TO_STATE(fault_p->px_fh_dip); 2180 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 2181 px_cb_t *cb_p = (px_cb_t *)px_get_cb(fault_p->px_fh_dip); 2182 px_cb_list_t *pxl, *pxl_new; 2183 boolean_t is_proxy = B_FALSE; 2184 2185 /* create cb */ 2186 if (cb_p == NULL) { 2187 cb_p = kmem_zalloc(sizeof (px_cb_t), KM_SLEEP); 2188 2189 mutex_init(&cb_p->cb_mutex, NULL, MUTEX_DRIVER, 2190 (void *) ipltospl(FM_ERR_PIL)); 2191 2192 cb_p->px_cb_func = px_cb_intr; 2193 pxu_p->px_cb_p = cb_p; 2194 px_set_cb(fault_p->px_fh_dip, (uint64_t)cb_p); 2195 2196 /* px_lib_dev_init allows only FIRE and OBERON */ 2197 px_err_reg_enable( 2198 (pxu_p->chip_type == PX_CHIP_FIRE) ? 2199 PX_ERR_JBC : PX_ERR_UBC, 2200 pxu_p->px_address[PX_REG_XBC]); 2201 } else 2202 pxu_p->px_cb_p = cb_p; 2203 2204 /* register cb interrupt */ 2205 VERIFY(add_ivintr(fault_p->px_fh_sysino, PX_ERR_PIL, 2206 (intrfunc)cb_p->px_cb_func, (caddr_t)cb_p, NULL, NULL) == 0); 2207 2208 2209 /* update cb list */ 2210 mutex_enter(&cb_p->cb_mutex); 2211 if (cb_p->pxl == NULL) { 2212 is_proxy = B_TRUE; 2213 pxl = kmem_zalloc(sizeof (px_cb_list_t), KM_SLEEP); 2214 pxl->pxp = px_p; 2215 cb_p->pxl = pxl; 2216 cb_p->sysino = fault_p->px_fh_sysino; 2217 cb_p->cpuid = intr_dist_cpuid(); 2218 } else { 2219 /* 2220 * Find the last pxl or 2221 * stop short at encountering a redundent entry, or 2222 * both. 2223 */ 2224 pxl = cb_p->pxl; 2225 for (; !(pxl->pxp == px_p) && pxl->next; pxl = pxl->next) {}; 2226 ASSERT(pxl->pxp != px_p); 2227 2228 /* add to linked list */ 2229 pxl_new = kmem_zalloc(sizeof (px_cb_list_t), KM_SLEEP); 2230 pxl_new->pxp = px_p; 2231 pxl->next = pxl_new; 2232 } 2233 cb_p->attachcnt++; 2234 mutex_exit(&cb_p->cb_mutex); 2235 2236 if (is_proxy) { 2237 /* add to interrupt redistribution list */ 2238 intr_dist_add(px_cb_intr_redist, cb_p); 2239 2240 /* enable cb hw interrupt */ 2241 px_ib_intr_enable(px_p, cb_p->cpuid, fault_p->px_intr_ino); 2242 } 2243 2244 return (DDI_SUCCESS); 2245 } 2246 2247 /* 2248 * px_cb_rem_intr() - Called from detach(9E) to remove its CB 2249 * interrupt vector, to shift proxy to the next available px, 2250 * or disable CB interrupt when itself is the last. 2251 */ 2252 void 2253 px_cb_rem_intr(px_fault_t *fault_p) 2254 { 2255 px_t *px_p = DIP_TO_STATE(fault_p->px_fh_dip), *pxp; 2256 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 2257 px_cb_t *cb_p = PX2CB(px_p); 2258 px_cb_list_t *pxl, *prev; 2259 px_fault_t *f_p; 2260 2261 ASSERT(cb_p->pxl); 2262 2263 /* find and remove this px, and update cb list */ 2264 mutex_enter(&cb_p->cb_mutex); 2265 2266 pxl = cb_p->pxl; 2267 if (pxl->pxp == px_p) { 2268 cb_p->pxl = pxl->next; 2269 } else { 2270 prev = pxl; 2271 pxl = pxl->next; 2272 for (; pxl && (pxl->pxp != px_p); prev = pxl, pxl = pxl->next) { 2273 }; 2274 if (pxl == NULL) { 2275 cmn_err(CE_WARN, "px_cb_rem_intr: can't find px_p 0x%p " 2276 "in registered CB list.", (void *)px_p); 2277 mutex_exit(&cb_p->cb_mutex); 2278 return; 2279 } 2280 prev->next = pxl->next; 2281 } 2282 pxu_p->px_cb_p = NULL; 2283 cb_p->attachcnt--; 2284 kmem_free(pxl, sizeof (px_cb_list_t)); 2285 mutex_exit(&cb_p->cb_mutex); 2286 2287 /* disable cb hw interrupt */ 2288 if (fault_p->px_fh_sysino == cb_p->sysino) 2289 px_ib_intr_disable(px_p->px_ib_p, fault_p->px_intr_ino, 2290 IB_INTR_WAIT); 2291 2292 /* if last px, remove from interrupt redistribution list */ 2293 if (cb_p->pxl == NULL) 2294 intr_dist_rem(px_cb_intr_redist, cb_p); 2295 2296 /* de-register interrupt */ 2297 VERIFY(rem_ivintr(fault_p->px_fh_sysino, PX_ERR_PIL) == 0); 2298 2299 /* if not last px, assign next px to manage cb */ 2300 mutex_enter(&cb_p->cb_mutex); 2301 if (cb_p->pxl) { 2302 if (fault_p->px_fh_sysino == cb_p->sysino) { 2303 pxp = cb_p->pxl->pxp; 2304 f_p = &pxp->px_cb_fault; 2305 cb_p->sysino = f_p->px_fh_sysino; 2306 2307 PX_INTR_ENABLE(pxp->px_dip, cb_p->sysino, cb_p->cpuid); 2308 (void) px_lib_intr_setstate(pxp->px_dip, cb_p->sysino, 2309 INTR_IDLE_STATE); 2310 } 2311 mutex_exit(&cb_p->cb_mutex); 2312 return; 2313 } 2314 2315 /* clean up after the last px */ 2316 mutex_exit(&cb_p->cb_mutex); 2317 2318 /* px_lib_dev_init allows only FIRE and OBERON */ 2319 px_err_reg_disable( 2320 (pxu_p->chip_type == PX_CHIP_FIRE) ? PX_ERR_JBC : PX_ERR_UBC, 2321 pxu_p->px_address[PX_REG_XBC]); 2322 2323 mutex_destroy(&cb_p->cb_mutex); 2324 px_set_cb(fault_p->px_fh_dip, 0ull); 2325 kmem_free(cb_p, sizeof (px_cb_t)); 2326 } 2327 2328 /* 2329 * px_cb_intr() - sun4u only, CB interrupt dispatcher 2330 */ 2331 uint_t 2332 px_cb_intr(caddr_t arg) 2333 { 2334 px_cb_t *cb_p = (px_cb_t *)arg; 2335 px_t *pxp; 2336 px_fault_t *f_p; 2337 int ret; 2338 2339 mutex_enter(&cb_p->cb_mutex); 2340 2341 if (cb_p->pxl == NULL) { 2342 mutex_exit(&cb_p->cb_mutex); 2343 return (DDI_INTR_UNCLAIMED); 2344 } 2345 2346 pxp = cb_p->pxl->pxp; 2347 f_p = &pxp->px_cb_fault; 2348 2349 ret = f_p->px_err_func((caddr_t)f_p); 2350 2351 mutex_exit(&cb_p->cb_mutex); 2352 return (ret); 2353 } 2354 2355 #ifdef FMA 2356 void 2357 px_fill_rc_status(px_fault_t *px_fault_p, pciex_rc_error_regs_t *rc_status) 2358 { 2359 /* populate the rc_status by reading the registers - TBD */ 2360 } 2361 #endif /* FMA */ 2362 2363 /* 2364 * cpr callback 2365 * 2366 * disable fabric error msg interrupt prior to suspending 2367 * all device drivers; re-enable fabric error msg interrupt 2368 * after all devices are resumed. 2369 */ 2370 static boolean_t 2371 px_cpr_callb(void *arg, int code) 2372 { 2373 px_t *px_p = (px_t *)arg; 2374 px_ib_t *ib_p = px_p->px_ib_p; 2375 px_pec_t *pec_p = px_p->px_pec_p; 2376 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 2377 caddr_t csr_base; 2378 devino_t ce_ino, nf_ino, f_ino; 2379 px_ino_t *ce_ino_p, *nf_ino_p, *f_ino_p; 2380 uint64_t imu_log_enable, imu_intr_enable; 2381 uint64_t imu_log_mask, imu_intr_mask; 2382 2383 ce_ino = px_msiqid_to_devino(px_p, pec_p->pec_corr_msg_msiq_id); 2384 nf_ino = px_msiqid_to_devino(px_p, pec_p->pec_non_fatal_msg_msiq_id); 2385 f_ino = px_msiqid_to_devino(px_p, pec_p->pec_fatal_msg_msiq_id); 2386 csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR]; 2387 2388 imu_log_enable = CSR_XR(csr_base, IMU_ERROR_LOG_ENABLE); 2389 imu_intr_enable = CSR_XR(csr_base, IMU_INTERRUPT_ENABLE); 2390 2391 imu_log_mask = BITMASK(IMU_ERROR_LOG_ENABLE_FATAL_MES_NOT_EN_LOG_EN) | 2392 BITMASK(IMU_ERROR_LOG_ENABLE_NONFATAL_MES_NOT_EN_LOG_EN) | 2393 BITMASK(IMU_ERROR_LOG_ENABLE_COR_MES_NOT_EN_LOG_EN); 2394 2395 imu_intr_mask = 2396 BITMASK(IMU_INTERRUPT_ENABLE_FATAL_MES_NOT_EN_S_INT_EN) | 2397 BITMASK(IMU_INTERRUPT_ENABLE_NONFATAL_MES_NOT_EN_S_INT_EN) | 2398 BITMASK(IMU_INTERRUPT_ENABLE_COR_MES_NOT_EN_S_INT_EN) | 2399 BITMASK(IMU_INTERRUPT_ENABLE_FATAL_MES_NOT_EN_P_INT_EN) | 2400 BITMASK(IMU_INTERRUPT_ENABLE_NONFATAL_MES_NOT_EN_P_INT_EN) | 2401 BITMASK(IMU_INTERRUPT_ENABLE_COR_MES_NOT_EN_P_INT_EN); 2402 2403 switch (code) { 2404 case CB_CODE_CPR_CHKPT: 2405 /* disable imu rbne on corr/nonfatal/fatal errors */ 2406 CSR_XS(csr_base, IMU_ERROR_LOG_ENABLE, 2407 imu_log_enable & (~imu_log_mask)); 2408 2409 CSR_XS(csr_base, IMU_INTERRUPT_ENABLE, 2410 imu_intr_enable & (~imu_intr_mask)); 2411 2412 /* disable CORR intr mapping */ 2413 px_ib_intr_disable(ib_p, ce_ino, IB_INTR_NOWAIT); 2414 2415 /* disable NON FATAL intr mapping */ 2416 px_ib_intr_disable(ib_p, nf_ino, IB_INTR_NOWAIT); 2417 2418 /* disable FATAL intr mapping */ 2419 px_ib_intr_disable(ib_p, f_ino, IB_INTR_NOWAIT); 2420 2421 break; 2422 2423 case CB_CODE_CPR_RESUME: 2424 pxu_p->cpr_flag = PX_NOT_CPR; 2425 mutex_enter(&ib_p->ib_ino_lst_mutex); 2426 2427 ce_ino_p = px_ib_locate_ino(ib_p, ce_ino); 2428 nf_ino_p = px_ib_locate_ino(ib_p, nf_ino); 2429 f_ino_p = px_ib_locate_ino(ib_p, f_ino); 2430 2431 /* enable CORR intr mapping */ 2432 if (ce_ino_p) 2433 px_ib_intr_enable(px_p, ce_ino_p->ino_cpuid, ce_ino); 2434 else 2435 cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to " 2436 "reenable PCIe Correctable msg intr.\n"); 2437 2438 /* enable NON FATAL intr mapping */ 2439 if (nf_ino_p) 2440 px_ib_intr_enable(px_p, nf_ino_p->ino_cpuid, nf_ino); 2441 else 2442 cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to " 2443 "reenable PCIe Non Fatal msg intr.\n"); 2444 2445 /* enable FATAL intr mapping */ 2446 if (f_ino_p) 2447 px_ib_intr_enable(px_p, f_ino_p->ino_cpuid, f_ino); 2448 else 2449 cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to " 2450 "reenable PCIe Fatal msg intr.\n"); 2451 2452 mutex_exit(&ib_p->ib_ino_lst_mutex); 2453 2454 /* enable corr/nonfatal/fatal not enable error */ 2455 CSR_XS(csr_base, IMU_ERROR_LOG_ENABLE, (imu_log_enable | 2456 (imu_log_mask & px_imu_log_mask))); 2457 CSR_XS(csr_base, IMU_INTERRUPT_ENABLE, (imu_intr_enable | 2458 (imu_intr_mask & px_imu_intr_mask))); 2459 2460 break; 2461 } 2462 2463 return (B_TRUE); 2464 } 2465 2466 uint64_t 2467 px_get_rng_parent_hi_mask(px_t *px_p) 2468 { 2469 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 2470 uint64_t mask; 2471 2472 switch (PX_CHIP_TYPE(pxu_p)) { 2473 case PX_CHIP_OBERON: 2474 mask = OBERON_RANGE_PROP_MASK; 2475 break; 2476 case PX_CHIP_FIRE: 2477 mask = PX_RANGE_PROP_MASK; 2478 break; 2479 default: 2480 mask = PX_RANGE_PROP_MASK; 2481 } 2482 2483 return (mask); 2484 } 2485 2486 /* 2487 * fetch chip's range propery's value 2488 */ 2489 uint64_t 2490 px_get_range_prop(px_t *px_p, pci_ranges_t *rp, int bank) 2491 { 2492 uint64_t mask, range_prop; 2493 2494 mask = px_get_rng_parent_hi_mask(px_p); 2495 range_prop = (((uint64_t)(rp[bank].parent_high & mask)) << 32) | 2496 rp[bank].parent_low; 2497 2498 return (range_prop); 2499 } 2500 2501 /* 2502 * fetch the config space base addr of the root complex 2503 * note this depends on px structure being initialized 2504 */ 2505 uint64_t 2506 px_lib_get_cfgacc_base(dev_info_t *dip) 2507 { 2508 int instance = DIP_TO_INST(dip); 2509 px_t *px_p = INST_TO_STATE(instance); 2510 pci_ranges_t *rp = px_p->px_ranges_p; 2511 int bank = PCI_REG_ADDR_G(PCI_ADDR_CONFIG); 2512 2513 /* Get Fire's Physical Base Address */ 2514 return (px_get_range_prop(px_p, rp, bank)); 2515 } 2516 2517 /* 2518 * add cpr callback 2519 */ 2520 void 2521 px_cpr_add_callb(px_t *px_p) 2522 { 2523 px_p->px_cprcb_id = callb_add(px_cpr_callb, (void *)px_p, 2524 CB_CL_CPR_POST_USER, "px_cpr"); 2525 } 2526 2527 /* 2528 * remove cpr callback 2529 */ 2530 void 2531 px_cpr_rem_callb(px_t *px_p) 2532 { 2533 (void) callb_delete(px_p->px_cprcb_id); 2534 } 2535 2536 /*ARGSUSED*/ 2537 static uint_t 2538 px_hp_intr(caddr_t arg1, caddr_t arg2) 2539 { 2540 px_t *px_p = (px_t *)arg1; 2541 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 2542 int rval; 2543 2544 rval = pcie_intr(px_p->px_dip); 2545 2546 #ifdef DEBUG 2547 if (rval == DDI_INTR_UNCLAIMED) 2548 cmn_err(CE_WARN, "%s%d: UNCLAIMED intr\n", 2549 ddi_driver_name(px_p->px_dip), 2550 ddi_get_instance(px_p->px_dip)); 2551 #endif 2552 2553 /* Set the interrupt state to idle */ 2554 if (px_lib_intr_setstate(px_p->px_dip, 2555 pxu_p->hp_sysino, INTR_IDLE_STATE) != DDI_SUCCESS) 2556 return (DDI_INTR_UNCLAIMED); 2557 2558 return (rval); 2559 } 2560 2561 int 2562 px_lib_hotplug_init(dev_info_t *dip, void *arg) 2563 { 2564 px_t *px_p = DIP_TO_STATE(dip); 2565 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 2566 uint64_t ret; 2567 2568 if (ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 2569 "hotplug-capable") == 0) 2570 return (DDI_FAILURE); 2571 2572 if ((ret = hvio_hotplug_init(dip, arg)) == DDI_SUCCESS) { 2573 if (px_lib_intr_devino_to_sysino(px_p->px_dip, 2574 px_p->px_inos[PX_INTR_HOTPLUG], &pxu_p->hp_sysino) != 2575 DDI_SUCCESS) { 2576 #ifdef DEBUG 2577 cmn_err(CE_WARN, "%s%d: devino_to_sysino fails\n", 2578 ddi_driver_name(px_p->px_dip), 2579 ddi_get_instance(px_p->px_dip)); 2580 #endif 2581 return (DDI_FAILURE); 2582 } 2583 2584 VERIFY(add_ivintr(pxu_p->hp_sysino, PCIE_INTR_PRI, 2585 (intrfunc)px_hp_intr, (caddr_t)px_p, NULL, NULL) == 0); 2586 2587 px_ib_intr_enable(px_p, intr_dist_cpuid(), 2588 px_p->px_inos[PX_INTR_HOTPLUG]); 2589 } 2590 2591 return (ret); 2592 } 2593 2594 void 2595 px_lib_hotplug_uninit(dev_info_t *dip) 2596 { 2597 if (hvio_hotplug_uninit(dip) == DDI_SUCCESS) { 2598 px_t *px_p = DIP_TO_STATE(dip); 2599 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 2600 2601 px_ib_intr_disable(px_p->px_ib_p, 2602 px_p->px_inos[PX_INTR_HOTPLUG], IB_INTR_WAIT); 2603 2604 VERIFY(rem_ivintr(pxu_p->hp_sysino, PCIE_INTR_PRI) == 0); 2605 } 2606 } 2607 2608 /* 2609 * px_hp_intr_redist() - sun4u only, HP interrupt redistribution 2610 */ 2611 void 2612 px_hp_intr_redist(px_t *px_p) 2613 { 2614 pcie_bus_t *bus_p = PCIE_DIP2BUS(px_p->px_dip); 2615 2616 if (px_p && PCIE_IS_PCIE_HOTPLUG_ENABLED(bus_p)) { 2617 px_ib_intr_dist_en(px_p->px_dip, intr_dist_cpuid(), 2618 px_p->px_inos[PX_INTR_HOTPLUG], B_FALSE); 2619 } 2620 } 2621 2622 boolean_t 2623 px_lib_is_in_drain_state(px_t *px_p) 2624 { 2625 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 2626 caddr_t csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR]; 2627 uint64_t drain_status; 2628 2629 if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON) { 2630 drain_status = CSR_BR(csr_base, DRAIN_CONTROL_STATUS, DRAIN); 2631 } else { 2632 drain_status = CSR_BR(csr_base, TLU_STATUS, DRAIN); 2633 } 2634 2635 return (drain_status); 2636 } 2637 2638 pcie_req_id_t 2639 px_lib_get_bdf(px_t *px_p) 2640 { 2641 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 2642 caddr_t csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR]; 2643 pcie_req_id_t bdf; 2644 2645 bdf = CSR_BR(csr_base, DMC_PCI_EXPRESS_CONFIGURATION, REQ_ID); 2646 2647 return (bdf); 2648 } 2649 2650 /*ARGSUSED*/ 2651 int 2652 px_lib_get_root_complex_mps(px_t *px_p, dev_info_t *dip, int *mps) 2653 { 2654 pxu_t *pxu_p; 2655 caddr_t csr_base; 2656 2657 pxu_p = (pxu_t *)px_p->px_plat_p; 2658 2659 if (pxu_p == NULL) 2660 return (DDI_FAILURE); 2661 2662 csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR]; 2663 2664 2665 *mps = CSR_XR(csr_base, TLU_DEVICE_CAPABILITIES) & 2666 TLU_DEVICE_CAPABILITIES_MPS_MASK; 2667 2668 return (DDI_SUCCESS); 2669 } 2670 2671 /*ARGSUSED*/ 2672 int 2673 px_lib_set_root_complex_mps(px_t *px_p, dev_info_t *dip, int mps) 2674 { 2675 pxu_t *pxu_p; 2676 caddr_t csr_base; 2677 uint64_t dev_ctrl; 2678 int link_width, val; 2679 px_chip_type_t chip_type = px_identity_init(px_p); 2680 2681 pxu_p = (pxu_t *)px_p->px_plat_p; 2682 2683 if (pxu_p == NULL) 2684 return (DDI_FAILURE); 2685 2686 csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR]; 2687 2688 dev_ctrl = CSR_XR(csr_base, TLU_DEVICE_CONTROL); 2689 dev_ctrl |= (mps << TLU_DEVICE_CONTROL_MPS); 2690 2691 CSR_XS(csr_base, TLU_DEVICE_CONTROL, dev_ctrl); 2692 2693 link_width = CSR_FR(csr_base, TLU_LINK_STATUS, WIDTH); 2694 2695 /* 2696 * Convert link_width to match timer array configuration. 2697 */ 2698 switch (link_width) { 2699 case 1: 2700 link_width = 0; 2701 break; 2702 case 4: 2703 link_width = 1; 2704 break; 2705 case 8: 2706 link_width = 2; 2707 break; 2708 case 16: 2709 link_width = 3; 2710 break; 2711 default: 2712 link_width = 0; 2713 } 2714 2715 val = px_replay_timer_table[mps][link_width]; 2716 CSR_XS(csr_base, LPU_TXLINK_REPLAY_TIMER_THRESHOLD, val); 2717 2718 if (chip_type == PX_CHIP_OBERON) 2719 return (DDI_SUCCESS); 2720 2721 val = px_acknak_timer_table[mps][link_width]; 2722 CSR_XS(csr_base, LPU_TXLINK_FREQUENT_NAK_LATENCY_TIMER_THRESHOLD, val); 2723 2724 return (DDI_SUCCESS); 2725 } 2726 2727 /*ARGSUSED*/ 2728 int 2729 px_lib_fabric_sync(dev_info_t *dip) 2730 { 2731 /* an no-op on sun4u platform */ 2732 return (DDI_SUCCESS); 2733 } 2734