1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/types.h> 30 #include <sys/kmem.h> 31 #include <sys/conf.h> 32 #include <sys/ddi.h> 33 #include <sys/sunddi.h> 34 #include <sys/fm/protocol.h> 35 #include <sys/fm/util.h> 36 #include <sys/modctl.h> 37 #include <sys/disp.h> 38 #include <sys/stat.h> 39 #include <sys/ddi_impldefs.h> 40 #include <sys/vmem.h> 41 #include <sys/iommutsb.h> 42 #include <sys/cpuvar.h> 43 #include <sys/ivintr.h> 44 #include <sys/byteorder.h> 45 #include <px_obj.h> 46 #include <pcie_pwr.h> 47 #include "px_tools_var.h" 48 #include <px_regs.h> 49 #include <px_csr.h> 50 #include <sys/machsystm.h> 51 #include "px_lib4u.h" 52 #include "px_err.h" 53 54 #pragma weak jbus_stst_order 55 56 extern void jbus_stst_order(); 57 58 ulong_t px_mmu_dvma_end = 0xfffffffful; 59 uint_t px_ranges_phi_mask = 0xfffffffful; 60 61 static int px_goto_l23ready(px_t *px_p); 62 static int px_goto_l0(px_t *px_p); 63 static int px_pre_pwron_check(px_t *px_p); 64 static uint32_t px_identity_chip(px_t *px_p); 65 static void px_lib_clr_errs(px_t *px_p, px_pec_t *pec_p); 66 static boolean_t px_cpr_callb(void *arg, int code); 67 68 /* 69 * px_lib_map_registers 70 * 71 * This function is called from the attach routine to map the registers 72 * accessed by this driver. 73 * 74 * used by: px_attach() 75 * 76 * return value: DDI_FAILURE on failure 77 */ 78 int 79 px_lib_map_regs(pxu_t *pxu_p, dev_info_t *dip) 80 { 81 ddi_device_acc_attr_t attr; 82 px_reg_bank_t reg_bank = PX_REG_CSR; 83 84 DBG(DBG_ATTACH, dip, "px_lib_map_regs: pxu_p:0x%p, dip 0x%p\n", 85 pxu_p, dip); 86 87 attr.devacc_attr_version = DDI_DEVICE_ATTR_V0; 88 attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC; 89 attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 90 91 /* 92 * PCI CSR Base 93 */ 94 if (ddi_regs_map_setup(dip, reg_bank, &pxu_p->px_address[reg_bank], 95 0, 0, &attr, &pxu_p->px_ac[reg_bank]) != DDI_SUCCESS) { 96 goto fail; 97 } 98 99 reg_bank++; 100 101 /* 102 * XBUS CSR Base 103 */ 104 if (ddi_regs_map_setup(dip, reg_bank, &pxu_p->px_address[reg_bank], 105 0, 0, &attr, &pxu_p->px_ac[reg_bank]) != DDI_SUCCESS) { 106 goto fail; 107 } 108 109 pxu_p->px_address[reg_bank] -= FIRE_CONTROL_STATUS; 110 111 done: 112 for (; reg_bank >= PX_REG_CSR; reg_bank--) { 113 DBG(DBG_ATTACH, dip, "reg_bank 0x%x address 0x%p\n", 114 reg_bank, pxu_p->px_address[reg_bank]); 115 } 116 117 return (DDI_SUCCESS); 118 119 fail: 120 cmn_err(CE_WARN, "%s%d: unable to map reg entry %d\n", 121 ddi_driver_name(dip), ddi_get_instance(dip), reg_bank); 122 123 for (reg_bank--; reg_bank >= PX_REG_CSR; reg_bank--) { 124 pxu_p->px_address[reg_bank] = NULL; 125 ddi_regs_map_free(&pxu_p->px_ac[reg_bank]); 126 } 127 128 return (DDI_FAILURE); 129 } 130 131 /* 132 * px_lib_unmap_regs: 133 * 134 * This routine unmaps the registers mapped by map_px_registers. 135 * 136 * used by: px_detach(), and error conditions in px_attach() 137 * 138 * return value: none 139 */ 140 void 141 px_lib_unmap_regs(pxu_t *pxu_p) 142 { 143 int i; 144 145 for (i = 0; i < PX_REG_MAX; i++) { 146 if (pxu_p->px_ac[i]) 147 ddi_regs_map_free(&pxu_p->px_ac[i]); 148 } 149 } 150 151 int 152 px_lib_dev_init(dev_info_t *dip, devhandle_t *dev_hdl) 153 { 154 px_t *px_p = DIP_TO_STATE(dip); 155 caddr_t xbc_csr_base, csr_base; 156 px_dvma_range_prop_t px_dvma_range; 157 uint32_t chip_id; 158 pxu_t *pxu_p; 159 160 DBG(DBG_ATTACH, dip, "px_lib_dev_init: dip 0x%p\n", dip); 161 162 if ((chip_id = px_identity_chip(px_p)) == PX_CHIP_UNIDENTIFIED) 163 return (DDI_FAILURE); 164 165 switch (chip_id) { 166 case FIRE_VER_10: 167 cmn_err(CE_WARN, "FIRE Hardware Version 1.0 is not supported"); 168 return (DDI_FAILURE); 169 case FIRE_VER_20: 170 DBG(DBG_ATTACH, dip, "FIRE Hardware Version 2.0\n"); 171 break; 172 default: 173 cmn_err(CE_WARN, "%s%d: FIRE Hardware Version Unknown\n", 174 ddi_driver_name(dip), ddi_get_instance(dip)); 175 return (DDI_FAILURE); 176 } 177 178 /* 179 * Allocate platform specific structure and link it to 180 * the px state structure. 181 */ 182 pxu_p = kmem_zalloc(sizeof (pxu_t), KM_SLEEP); 183 pxu_p->chip_id = chip_id; 184 pxu_p->portid = ddi_getprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 185 "portid", -1); 186 187 /* Map in the registers */ 188 if (px_lib_map_regs(pxu_p, dip) == DDI_FAILURE) { 189 kmem_free(pxu_p, sizeof (pxu_t)); 190 191 return (DDI_FAILURE); 192 } 193 194 xbc_csr_base = (caddr_t)pxu_p->px_address[PX_REG_XBC]; 195 csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR]; 196 197 pxu_p->tsb_cookie = iommu_tsb_alloc(pxu_p->portid); 198 pxu_p->tsb_size = iommu_tsb_cookie_to_size(pxu_p->tsb_cookie); 199 pxu_p->tsb_vaddr = iommu_tsb_cookie_to_va(pxu_p->tsb_cookie); 200 201 /* 202 * Create "virtual-dma" property to support child devices 203 * needing to know DVMA range. 204 */ 205 px_dvma_range.dvma_base = (uint32_t)px_mmu_dvma_end + 1 206 - ((pxu_p->tsb_size >> 3) << MMU_PAGE_SHIFT); 207 px_dvma_range.dvma_len = (uint32_t) 208 px_mmu_dvma_end - px_dvma_range.dvma_base + 1; 209 210 (void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP, 211 "virtual-dma", (caddr_t)&px_dvma_range, 212 sizeof (px_dvma_range_prop_t)); 213 /* 214 * Initilize all fire hardware specific blocks. 215 */ 216 hvio_cb_init(xbc_csr_base, pxu_p); 217 hvio_ib_init(csr_base, pxu_p); 218 hvio_pec_init(csr_base, pxu_p); 219 hvio_mmu_init(csr_base, pxu_p); 220 221 px_p->px_plat_p = (void *)pxu_p; 222 223 /* 224 * Initialize all the interrupt handlers 225 */ 226 px_err_reg_enable(px_p, PX_ERR_JBC); 227 px_err_reg_enable(px_p, PX_ERR_MMU); 228 px_err_reg_enable(px_p, PX_ERR_IMU); 229 px_err_reg_enable(px_p, PX_ERR_TLU_UE); 230 px_err_reg_enable(px_p, PX_ERR_TLU_CE); 231 px_err_reg_enable(px_p, PX_ERR_TLU_OE); 232 px_err_reg_enable(px_p, PX_ERR_ILU); 233 px_err_reg_enable(px_p, PX_ERR_LPU_LINK); 234 px_err_reg_enable(px_p, PX_ERR_LPU_PHY); 235 px_err_reg_enable(px_p, PX_ERR_LPU_RX); 236 px_err_reg_enable(px_p, PX_ERR_LPU_TX); 237 px_err_reg_enable(px_p, PX_ERR_LPU_LTSSM); 238 px_err_reg_enable(px_p, PX_ERR_LPU_GIGABLZ); 239 240 /* Initilize device handle */ 241 *dev_hdl = (devhandle_t)csr_base; 242 243 DBG(DBG_ATTACH, dip, "px_lib_dev_init: dev_hdl 0x%llx\n", *dev_hdl); 244 245 return (DDI_SUCCESS); 246 } 247 248 int 249 px_lib_dev_fini(dev_info_t *dip) 250 { 251 px_t *px_p = DIP_TO_STATE(dip); 252 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 253 254 DBG(DBG_DETACH, dip, "px_lib_dev_fini: dip 0x%p\n", dip); 255 256 /* 257 * Deinitialize all the interrupt handlers 258 */ 259 px_err_reg_disable(px_p, PX_ERR_JBC); 260 px_err_reg_disable(px_p, PX_ERR_MMU); 261 px_err_reg_disable(px_p, PX_ERR_IMU); 262 px_err_reg_disable(px_p, PX_ERR_TLU_UE); 263 px_err_reg_disable(px_p, PX_ERR_TLU_CE); 264 px_err_reg_disable(px_p, PX_ERR_TLU_OE); 265 px_err_reg_disable(px_p, PX_ERR_ILU); 266 px_err_reg_disable(px_p, PX_ERR_LPU_LINK); 267 px_err_reg_disable(px_p, PX_ERR_LPU_PHY); 268 px_err_reg_disable(px_p, PX_ERR_LPU_RX); 269 px_err_reg_disable(px_p, PX_ERR_LPU_TX); 270 px_err_reg_disable(px_p, PX_ERR_LPU_LTSSM); 271 px_err_reg_disable(px_p, PX_ERR_LPU_GIGABLZ); 272 273 iommu_tsb_free(pxu_p->tsb_cookie); 274 275 px_lib_unmap_regs((pxu_t *)px_p->px_plat_p); 276 kmem_free(px_p->px_plat_p, sizeof (pxu_t)); 277 px_p->px_plat_p = NULL; 278 279 return (DDI_SUCCESS); 280 } 281 282 /*ARGSUSED*/ 283 int 284 px_lib_intr_devino_to_sysino(dev_info_t *dip, devino_t devino, 285 sysino_t *sysino) 286 { 287 px_t *px_p = DIP_TO_STATE(dip); 288 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 289 uint64_t ret; 290 291 DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: dip 0x%p " 292 "devino 0x%x\n", dip, devino); 293 294 if ((ret = hvio_intr_devino_to_sysino(DIP_TO_HANDLE(dip), 295 pxu_p, devino, sysino)) != H_EOK) { 296 DBG(DBG_LIB_INT, dip, 297 "hvio_intr_devino_to_sysino failed, ret 0x%lx\n", ret); 298 return (DDI_FAILURE); 299 } 300 301 DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: sysino 0x%llx\n", 302 *sysino); 303 304 return (DDI_SUCCESS); 305 } 306 307 /*ARGSUSED*/ 308 int 309 px_lib_intr_getvalid(dev_info_t *dip, sysino_t sysino, 310 intr_valid_state_t *intr_valid_state) 311 { 312 uint64_t ret; 313 314 DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: dip 0x%p sysino 0x%llx\n", 315 dip, sysino); 316 317 if ((ret = hvio_intr_getvalid(DIP_TO_HANDLE(dip), 318 sysino, intr_valid_state)) != H_EOK) { 319 DBG(DBG_LIB_INT, dip, "hvio_intr_getvalid failed, ret 0x%lx\n", 320 ret); 321 return (DDI_FAILURE); 322 } 323 324 DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: intr_valid_state 0x%x\n", 325 *intr_valid_state); 326 327 return (DDI_SUCCESS); 328 } 329 330 /*ARGSUSED*/ 331 int 332 px_lib_intr_setvalid(dev_info_t *dip, sysino_t sysino, 333 intr_valid_state_t intr_valid_state) 334 { 335 uint64_t ret; 336 337 DBG(DBG_LIB_INT, dip, "px_lib_intr_setvalid: dip 0x%p sysino 0x%llx " 338 "intr_valid_state 0x%x\n", dip, sysino, intr_valid_state); 339 340 if ((ret = hvio_intr_setvalid(DIP_TO_HANDLE(dip), 341 sysino, intr_valid_state)) != H_EOK) { 342 DBG(DBG_LIB_INT, dip, "hvio_intr_setvalid failed, ret 0x%lx\n", 343 ret); 344 return (DDI_FAILURE); 345 } 346 347 return (DDI_SUCCESS); 348 } 349 350 /*ARGSUSED*/ 351 int 352 px_lib_intr_getstate(dev_info_t *dip, sysino_t sysino, 353 intr_state_t *intr_state) 354 { 355 uint64_t ret; 356 357 DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: dip 0x%p sysino 0x%llx\n", 358 dip, sysino); 359 360 if ((ret = hvio_intr_getstate(DIP_TO_HANDLE(dip), 361 sysino, intr_state)) != H_EOK) { 362 DBG(DBG_LIB_INT, dip, "hvio_intr_getstate failed, ret 0x%lx\n", 363 ret); 364 return (DDI_FAILURE); 365 } 366 367 DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: intr_state 0x%x\n", 368 *intr_state); 369 370 return (DDI_SUCCESS); 371 } 372 373 /*ARGSUSED*/ 374 int 375 px_lib_intr_setstate(dev_info_t *dip, sysino_t sysino, 376 intr_state_t intr_state) 377 { 378 uint64_t ret; 379 380 DBG(DBG_LIB_INT, dip, "px_lib_intr_setstate: dip 0x%p sysino 0x%llx " 381 "intr_state 0x%x\n", dip, sysino, intr_state); 382 383 if ((ret = hvio_intr_setstate(DIP_TO_HANDLE(dip), 384 sysino, intr_state)) != H_EOK) { 385 DBG(DBG_LIB_INT, dip, "hvio_intr_setstate failed, ret 0x%lx\n", 386 ret); 387 return (DDI_FAILURE); 388 } 389 390 return (DDI_SUCCESS); 391 } 392 393 /*ARGSUSED*/ 394 int 395 px_lib_intr_gettarget(dev_info_t *dip, sysino_t sysino, cpuid_t *cpuid) 396 { 397 uint64_t ret; 398 399 DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: dip 0x%p sysino 0x%llx\n", 400 dip, sysino); 401 402 if ((ret = hvio_intr_gettarget(DIP_TO_HANDLE(dip), 403 sysino, cpuid)) != H_EOK) { 404 DBG(DBG_LIB_INT, dip, "hvio_intr_gettarget failed, ret 0x%lx\n", 405 ret); 406 return (DDI_FAILURE); 407 } 408 409 DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: cpuid 0x%x\n", cpuid); 410 411 return (DDI_SUCCESS); 412 } 413 414 /*ARGSUSED*/ 415 int 416 px_lib_intr_settarget(dev_info_t *dip, sysino_t sysino, cpuid_t cpuid) 417 { 418 uint64_t ret; 419 420 DBG(DBG_LIB_INT, dip, "px_lib_intr_settarget: dip 0x%p sysino 0x%llx " 421 "cpuid 0x%x\n", dip, sysino, cpuid); 422 423 if ((ret = hvio_intr_settarget(DIP_TO_HANDLE(dip), 424 sysino, cpuid)) != H_EOK) { 425 DBG(DBG_LIB_INT, dip, "hvio_intr_settarget failed, ret 0x%lx\n", 426 ret); 427 return (DDI_FAILURE); 428 } 429 430 return (DDI_SUCCESS); 431 } 432 433 /*ARGSUSED*/ 434 int 435 px_lib_intr_reset(dev_info_t *dip) 436 { 437 devino_t ino; 438 sysino_t sysino; 439 440 DBG(DBG_LIB_INT, dip, "px_lib_intr_reset: dip 0x%p\n", dip); 441 442 /* Reset all Interrupts */ 443 for (ino = 0; ino < INTERRUPT_MAPPING_ENTRIES; ino++) { 444 if (px_lib_intr_devino_to_sysino(dip, ino, 445 &sysino) != DDI_SUCCESS) 446 return (BF_FATAL); 447 448 if (px_lib_intr_setstate(dip, sysino, 449 INTR_IDLE_STATE) != DDI_SUCCESS) 450 return (BF_FATAL); 451 } 452 453 return (BF_NONE); 454 } 455 456 /*ARGSUSED*/ 457 int 458 px_lib_iommu_map(dev_info_t *dip, tsbid_t tsbid, pages_t pages, 459 io_attributes_t io_attributes, void *addr, size_t pfn_index, 460 int flag) 461 { 462 px_t *px_p = DIP_TO_STATE(dip); 463 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 464 uint64_t ret; 465 466 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: dip 0x%p tsbid 0x%llx " 467 "pages 0x%x atrr 0x%x addr 0x%p pfn_index 0x%llx, flag 0x%x\n", 468 dip, tsbid, pages, io_attributes, addr, pfn_index, flag); 469 470 if ((ret = hvio_iommu_map(px_p->px_dev_hdl, pxu_p, tsbid, pages, 471 io_attributes, addr, pfn_index, flag)) != H_EOK) { 472 DBG(DBG_LIB_DMA, dip, 473 "px_lib_iommu_map failed, ret 0x%lx\n", ret); 474 return (DDI_FAILURE); 475 } 476 477 return (DDI_SUCCESS); 478 } 479 480 /*ARGSUSED*/ 481 int 482 px_lib_iommu_demap(dev_info_t *dip, tsbid_t tsbid, pages_t pages) 483 { 484 px_t *px_p = DIP_TO_STATE(dip); 485 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 486 uint64_t ret; 487 488 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: dip 0x%p tsbid 0x%llx " 489 "pages 0x%x\n", dip, tsbid, pages); 490 491 if ((ret = hvio_iommu_demap(px_p->px_dev_hdl, pxu_p, tsbid, pages)) 492 != H_EOK) { 493 DBG(DBG_LIB_DMA, dip, 494 "px_lib_iommu_demap failed, ret 0x%lx\n", ret); 495 496 return (DDI_FAILURE); 497 } 498 499 return (DDI_SUCCESS); 500 } 501 502 /*ARGSUSED*/ 503 int 504 px_lib_iommu_getmap(dev_info_t *dip, tsbid_t tsbid, 505 io_attributes_t *attributes_p, r_addr_t *r_addr_p) 506 { 507 px_t *px_p = DIP_TO_STATE(dip); 508 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 509 uint64_t ret; 510 511 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: dip 0x%p tsbid 0x%llx\n", 512 dip, tsbid); 513 514 if ((ret = hvio_iommu_getmap(DIP_TO_HANDLE(dip), pxu_p, tsbid, 515 attributes_p, r_addr_p)) != H_EOK) { 516 DBG(DBG_LIB_DMA, dip, 517 "hvio_iommu_getmap failed, ret 0x%lx\n", ret); 518 519 return ((ret == H_ENOMAP) ? DDI_DMA_NOMAPPING:DDI_FAILURE); 520 } 521 522 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: attr 0x%x r_addr 0x%llx\n", 523 *attributes_p, *r_addr_p); 524 525 return (DDI_SUCCESS); 526 } 527 528 529 /* 530 * Checks dma attributes against system bypass ranges 531 * The bypass range is determined by the hardware. Return them so the 532 * common code can do generic checking against them. 533 */ 534 /*ARGSUSED*/ 535 int 536 px_lib_dma_bypass_rngchk(ddi_dma_attr_t *attrp, uint64_t *lo_p, uint64_t *hi_p) 537 { 538 *lo_p = MMU_BYPASS_BASE; 539 *hi_p = MMU_BYPASS_END; 540 541 return (DDI_SUCCESS); 542 } 543 544 545 /*ARGSUSED*/ 546 int 547 px_lib_iommu_getbypass(dev_info_t *dip, r_addr_t ra, 548 io_attributes_t io_attributes, io_addr_t *io_addr_p) 549 { 550 uint64_t ret; 551 552 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: dip 0x%p ra 0x%llx " 553 "attr 0x%x\n", dip, ra, io_attributes); 554 555 if ((ret = hvio_iommu_getbypass(DIP_TO_HANDLE(dip), ra, 556 io_attributes, io_addr_p)) != H_EOK) { 557 DBG(DBG_LIB_DMA, dip, 558 "hvio_iommu_getbypass failed, ret 0x%lx\n", ret); 559 return (DDI_FAILURE); 560 } 561 562 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: io_addr 0x%llx\n", 563 *io_addr_p); 564 565 return (DDI_SUCCESS); 566 } 567 568 /* 569 * bus dma sync entry point. 570 */ 571 /*ARGSUSED*/ 572 int 573 px_lib_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 574 off_t off, size_t len, uint_t cache_flags) 575 { 576 ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; 577 578 DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: dip 0x%p rdip 0x%p " 579 "handle 0x%llx off 0x%x len 0x%x flags 0x%x\n", 580 dip, rdip, handle, off, len, cache_flags); 581 582 /* 583 * jbus_stst_order is found only in certain cpu modules. 584 * Just return success if not present. 585 */ 586 if (&jbus_stst_order == NULL) 587 return (DDI_SUCCESS); 588 589 if (!(mp->dmai_flags & DMAI_FLAGS_INUSE)) { 590 cmn_err(CE_WARN, "%s%d: Unbound dma handle %p.", 591 ddi_driver_name(rdip), ddi_get_instance(rdip), (void *)mp); 592 593 return (DDI_FAILURE); 594 } 595 596 if (mp->dmai_flags & DMAI_FLAGS_NOSYNC) 597 return (DDI_SUCCESS); 598 599 /* 600 * No flush needed when sending data from memory to device. 601 * Nothing to do to "sync" memory to what device would already see. 602 */ 603 if (!(mp->dmai_rflags & DDI_DMA_READ) || 604 ((cache_flags & PX_DMA_SYNC_DDI_FLAGS) == DDI_DMA_SYNC_FORDEV)) 605 return (DDI_SUCCESS); 606 607 /* 608 * Perform necessary cpu workaround to ensure jbus ordering. 609 * CPU's internal "invalidate FIFOs" are flushed. 610 */ 611 612 #if !defined(lint) 613 kpreempt_disable(); 614 #endif 615 jbus_stst_order(); 616 #if !defined(lint) 617 kpreempt_enable(); 618 #endif 619 return (DDI_SUCCESS); 620 } 621 622 /* 623 * MSIQ Functions: 624 */ 625 /*ARGSUSED*/ 626 int 627 px_lib_msiq_init(dev_info_t *dip) 628 { 629 px_t *px_p = DIP_TO_STATE(dip); 630 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 631 px_msiq_state_t *msiq_state_p = &px_p->px_ib_p->ib_msiq_state; 632 caddr_t msiq_addr; 633 px_dvma_addr_t pg_index; 634 size_t size; 635 int ret; 636 637 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_init: dip 0x%p\n", dip); 638 639 /* 640 * Map the EQ memory into the Fire MMU (has to be 512KB aligned) 641 * and then initialize the base address register. 642 * 643 * Allocate entries from Fire IOMMU so that the resulting address 644 * is properly aligned. Calculate the index of the first allocated 645 * entry. Note: The size of the mapping is assumed to be a multiple 646 * of the page size. 647 */ 648 msiq_addr = (caddr_t)(((uint64_t)msiq_state_p->msiq_buf_p + 649 (MMU_PAGE_SIZE - 1)) >> MMU_PAGE_SHIFT << MMU_PAGE_SHIFT); 650 651 size = msiq_state_p->msiq_cnt * 652 msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t); 653 654 pxu_p->msiq_mapped_p = vmem_xalloc(px_p->px_mmu_p->mmu_dvma_map, 655 size, (512 * 1024), 0, 0, NULL, NULL, VM_NOSLEEP | VM_BESTFIT); 656 657 if (pxu_p->msiq_mapped_p == NULL) 658 return (DDI_FAILURE); 659 660 pg_index = MMU_PAGE_INDEX(px_p->px_mmu_p, 661 MMU_BTOP((ulong_t)pxu_p->msiq_mapped_p)); 662 663 if ((ret = px_lib_iommu_map(px_p->px_dip, PCI_TSBID(0, pg_index), 664 MMU_BTOP(size), PCI_MAP_ATTR_WRITE, (void *)msiq_addr, 0, 665 MMU_MAP_BUF)) != DDI_SUCCESS) { 666 DBG(DBG_LIB_MSIQ, dip, 667 "hvio_msiq_init failed, ret 0x%lx\n", ret); 668 669 (void) px_lib_msiq_fini(dip); 670 return (DDI_FAILURE); 671 } 672 673 (void) hvio_msiq_init(DIP_TO_HANDLE(dip), pxu_p); 674 675 return (DDI_SUCCESS); 676 } 677 678 /*ARGSUSED*/ 679 int 680 px_lib_msiq_fini(dev_info_t *dip) 681 { 682 px_t *px_p = DIP_TO_STATE(dip); 683 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 684 px_msiq_state_t *msiq_state_p = &px_p->px_ib_p->ib_msiq_state; 685 px_dvma_addr_t pg_index; 686 size_t size; 687 688 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_fini: dip 0x%p\n", dip); 689 690 /* 691 * Unmap and free the EQ memory that had been mapped 692 * into the Fire IOMMU. 693 */ 694 size = msiq_state_p->msiq_cnt * 695 msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t); 696 697 pg_index = MMU_PAGE_INDEX(px_p->px_mmu_p, 698 MMU_BTOP((ulong_t)pxu_p->msiq_mapped_p)); 699 700 (void) px_lib_iommu_demap(px_p->px_dip, 701 PCI_TSBID(0, pg_index), MMU_BTOP(size)); 702 703 /* Free the entries from the Fire MMU */ 704 vmem_xfree(px_p->px_mmu_p->mmu_dvma_map, 705 (void *)pxu_p->msiq_mapped_p, size); 706 707 return (DDI_SUCCESS); 708 } 709 710 /*ARGSUSED*/ 711 int 712 px_lib_msiq_info(dev_info_t *dip, msiqid_t msiq_id, r_addr_t *ra_p, 713 uint_t *msiq_rec_cnt_p) 714 { 715 px_t *px_p = DIP_TO_STATE(dip); 716 px_msiq_state_t *msiq_state_p = &px_p->px_ib_p->ib_msiq_state; 717 uint64_t *msiq_addr; 718 size_t msiq_size; 719 720 DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: dip 0x%p msiq_id 0x%x\n", 721 dip, msiq_id); 722 723 msiq_addr = (uint64_t *)(((uint64_t)msiq_state_p->msiq_buf_p + 724 (MMU_PAGE_SIZE - 1)) >> MMU_PAGE_SHIFT << MMU_PAGE_SHIFT); 725 msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t); 726 ra_p = (r_addr_t *)((caddr_t)msiq_addr + (msiq_id * msiq_size)); 727 728 *msiq_rec_cnt_p = msiq_state_p->msiq_rec_cnt; 729 730 DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: ra_p 0x%p msiq_rec_cnt 0x%x\n", 731 ra_p, *msiq_rec_cnt_p); 732 733 return (DDI_SUCCESS); 734 } 735 736 /*ARGSUSED*/ 737 int 738 px_lib_msiq_getvalid(dev_info_t *dip, msiqid_t msiq_id, 739 pci_msiq_valid_state_t *msiq_valid_state) 740 { 741 uint64_t ret; 742 743 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: dip 0x%p msiq_id 0x%x\n", 744 dip, msiq_id); 745 746 if ((ret = hvio_msiq_getvalid(DIP_TO_HANDLE(dip), 747 msiq_id, msiq_valid_state)) != H_EOK) { 748 DBG(DBG_LIB_MSIQ, dip, 749 "hvio_msiq_getvalid failed, ret 0x%lx\n", ret); 750 return (DDI_FAILURE); 751 } 752 753 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: msiq_valid_state 0x%x\n", 754 *msiq_valid_state); 755 756 return (DDI_SUCCESS); 757 } 758 759 /*ARGSUSED*/ 760 int 761 px_lib_msiq_setvalid(dev_info_t *dip, msiqid_t msiq_id, 762 pci_msiq_valid_state_t msiq_valid_state) 763 { 764 uint64_t ret; 765 766 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setvalid: dip 0x%p msiq_id 0x%x " 767 "msiq_valid_state 0x%x\n", dip, msiq_id, msiq_valid_state); 768 769 if ((ret = hvio_msiq_setvalid(DIP_TO_HANDLE(dip), 770 msiq_id, msiq_valid_state)) != H_EOK) { 771 DBG(DBG_LIB_MSIQ, dip, 772 "hvio_msiq_setvalid failed, ret 0x%lx\n", ret); 773 return (DDI_FAILURE); 774 } 775 776 return (DDI_SUCCESS); 777 } 778 779 /*ARGSUSED*/ 780 int 781 px_lib_msiq_getstate(dev_info_t *dip, msiqid_t msiq_id, 782 pci_msiq_state_t *msiq_state) 783 { 784 uint64_t ret; 785 786 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: dip 0x%p msiq_id 0x%x\n", 787 dip, msiq_id); 788 789 if ((ret = hvio_msiq_getstate(DIP_TO_HANDLE(dip), 790 msiq_id, msiq_state)) != H_EOK) { 791 DBG(DBG_LIB_MSIQ, dip, 792 "hvio_msiq_getstate failed, ret 0x%lx\n", ret); 793 return (DDI_FAILURE); 794 } 795 796 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: msiq_state 0x%x\n", 797 *msiq_state); 798 799 return (DDI_SUCCESS); 800 } 801 802 /*ARGSUSED*/ 803 int 804 px_lib_msiq_setstate(dev_info_t *dip, msiqid_t msiq_id, 805 pci_msiq_state_t msiq_state) 806 { 807 uint64_t ret; 808 809 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setstate: dip 0x%p msiq_id 0x%x " 810 "msiq_state 0x%x\n", dip, msiq_id, msiq_state); 811 812 if ((ret = hvio_msiq_setstate(DIP_TO_HANDLE(dip), 813 msiq_id, msiq_state)) != H_EOK) { 814 DBG(DBG_LIB_MSIQ, dip, 815 "hvio_msiq_setstate failed, ret 0x%lx\n", ret); 816 return (DDI_FAILURE); 817 } 818 819 return (DDI_SUCCESS); 820 } 821 822 /*ARGSUSED*/ 823 int 824 px_lib_msiq_gethead(dev_info_t *dip, msiqid_t msiq_id, 825 msiqhead_t *msiq_head) 826 { 827 uint64_t ret; 828 829 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: dip 0x%p msiq_id 0x%x\n", 830 dip, msiq_id); 831 832 if ((ret = hvio_msiq_gethead(DIP_TO_HANDLE(dip), 833 msiq_id, msiq_head)) != H_EOK) { 834 DBG(DBG_LIB_MSIQ, dip, 835 "hvio_msiq_gethead failed, ret 0x%lx\n", ret); 836 return (DDI_FAILURE); 837 } 838 839 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: msiq_head 0x%x\n", 840 *msiq_head); 841 842 return (DDI_SUCCESS); 843 } 844 845 /*ARGSUSED*/ 846 int 847 px_lib_msiq_sethead(dev_info_t *dip, msiqid_t msiq_id, 848 msiqhead_t msiq_head) 849 { 850 uint64_t ret; 851 852 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_sethead: dip 0x%p msiq_id 0x%x " 853 "msiq_head 0x%x\n", dip, msiq_id, msiq_head); 854 855 if ((ret = hvio_msiq_sethead(DIP_TO_HANDLE(dip), 856 msiq_id, msiq_head)) != H_EOK) { 857 DBG(DBG_LIB_MSIQ, dip, 858 "hvio_msiq_sethead failed, ret 0x%lx\n", ret); 859 return (DDI_FAILURE); 860 } 861 862 return (DDI_SUCCESS); 863 } 864 865 /*ARGSUSED*/ 866 int 867 px_lib_msiq_gettail(dev_info_t *dip, msiqid_t msiq_id, 868 msiqtail_t *msiq_tail) 869 { 870 uint64_t ret; 871 872 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: dip 0x%p msiq_id 0x%x\n", 873 dip, msiq_id); 874 875 if ((ret = hvio_msiq_gettail(DIP_TO_HANDLE(dip), 876 msiq_id, msiq_tail)) != H_EOK) { 877 DBG(DBG_LIB_MSIQ, dip, 878 "hvio_msiq_gettail failed, ret 0x%lx\n", ret); 879 return (DDI_FAILURE); 880 } 881 882 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: msiq_tail 0x%x\n", 883 *msiq_tail); 884 885 return (DDI_SUCCESS); 886 } 887 888 /*ARGSUSED*/ 889 void 890 px_lib_get_msiq_rec(dev_info_t *dip, px_msiq_t *msiq_p, msiq_rec_t *msiq_rec_p) 891 { 892 eq_rec_t *eq_rec_p = (eq_rec_t *)msiq_p->msiq_curr; 893 894 DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: dip 0x%p eq_rec_p 0x%p\n", 895 dip, eq_rec_p); 896 897 if (!eq_rec_p->eq_rec_fmt_type) { 898 /* Set msiq_rec_type to zero */ 899 msiq_rec_p->msiq_rec_type = 0; 900 901 return; 902 } 903 904 DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: EQ RECORD, " 905 "eq_rec_rid 0x%llx eq_rec_fmt_type 0x%llx " 906 "eq_rec_len 0x%llx eq_rec_addr0 0x%llx " 907 "eq_rec_addr1 0x%llx eq_rec_data0 0x%llx " 908 "eq_rec_data1 0x%llx\n", eq_rec_p->eq_rec_rid, 909 eq_rec_p->eq_rec_fmt_type, eq_rec_p->eq_rec_len, 910 eq_rec_p->eq_rec_addr0, eq_rec_p->eq_rec_addr1, 911 eq_rec_p->eq_rec_data0, eq_rec_p->eq_rec_data1); 912 913 /* 914 * Only upper 4 bits of eq_rec_fmt_type is used 915 * to identify the EQ record type. 916 */ 917 switch (eq_rec_p->eq_rec_fmt_type >> 3) { 918 case EQ_REC_MSI32: 919 msiq_rec_p->msiq_rec_type = MSI32_REC; 920 921 msiq_rec_p->msiq_rec_data.msi.msi_data = 922 eq_rec_p->eq_rec_data0; 923 break; 924 case EQ_REC_MSI64: 925 msiq_rec_p->msiq_rec_type = MSI64_REC; 926 927 msiq_rec_p->msiq_rec_data.msi.msi_data = 928 eq_rec_p->eq_rec_data0; 929 break; 930 case EQ_REC_MSG: 931 msiq_rec_p->msiq_rec_type = MSG_REC; 932 933 msiq_rec_p->msiq_rec_data.msg.msg_route = 934 eq_rec_p->eq_rec_fmt_type & 7; 935 msiq_rec_p->msiq_rec_data.msg.msg_targ = eq_rec_p->eq_rec_rid; 936 msiq_rec_p->msiq_rec_data.msg.msg_code = eq_rec_p->eq_rec_data0; 937 break; 938 default: 939 cmn_err(CE_WARN, "%s%d: px_lib_get_msiq_rec: " 940 "0x%lx is an unknown EQ record type", 941 ddi_driver_name(dip), ddi_get_instance(dip), 942 eq_rec_p->eq_rec_fmt_type); 943 break; 944 } 945 946 msiq_rec_p->msiq_rec_rid = eq_rec_p->eq_rec_rid; 947 msiq_rec_p->msiq_rec_msi_addr = ((eq_rec_p->eq_rec_addr1 << 16) | 948 (eq_rec_p->eq_rec_addr0 << 2)); 949 950 /* Zero out eq_rec_fmt_type field */ 951 eq_rec_p->eq_rec_fmt_type = 0; 952 } 953 954 /* 955 * MSI Functions: 956 */ 957 /*ARGSUSED*/ 958 int 959 px_lib_msi_init(dev_info_t *dip) 960 { 961 px_t *px_p = DIP_TO_STATE(dip); 962 px_msi_state_t *msi_state_p = &px_p->px_ib_p->ib_msi_state; 963 uint64_t ret; 964 965 DBG(DBG_LIB_MSI, dip, "px_lib_msi_init: dip 0x%p\n", dip); 966 967 if ((ret = hvio_msi_init(DIP_TO_HANDLE(dip), 968 msi_state_p->msi_addr32, msi_state_p->msi_addr64)) != H_EOK) { 969 DBG(DBG_LIB_MSIQ, dip, "px_lib_msi_init failed, ret 0x%lx\n", 970 ret); 971 return (DDI_FAILURE); 972 } 973 974 return (DDI_SUCCESS); 975 } 976 977 /*ARGSUSED*/ 978 int 979 px_lib_msi_getmsiq(dev_info_t *dip, msinum_t msi_num, 980 msiqid_t *msiq_id) 981 { 982 uint64_t ret; 983 984 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: dip 0x%p msi_num 0x%x\n", 985 dip, msi_num); 986 987 if ((ret = hvio_msi_getmsiq(DIP_TO_HANDLE(dip), 988 msi_num, msiq_id)) != H_EOK) { 989 DBG(DBG_LIB_MSI, dip, 990 "hvio_msi_getmsiq failed, ret 0x%lx\n", ret); 991 return (DDI_FAILURE); 992 } 993 994 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: msiq_id 0x%x\n", 995 *msiq_id); 996 997 return (DDI_SUCCESS); 998 } 999 1000 /*ARGSUSED*/ 1001 int 1002 px_lib_msi_setmsiq(dev_info_t *dip, msinum_t msi_num, 1003 msiqid_t msiq_id, msi_type_t msitype) 1004 { 1005 uint64_t ret; 1006 1007 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setmsiq: dip 0x%p msi_num 0x%x " 1008 "msq_id 0x%x\n", dip, msi_num, msiq_id); 1009 1010 if ((ret = hvio_msi_setmsiq(DIP_TO_HANDLE(dip), 1011 msi_num, msiq_id)) != H_EOK) { 1012 DBG(DBG_LIB_MSI, dip, 1013 "hvio_msi_setmsiq failed, ret 0x%lx\n", ret); 1014 return (DDI_FAILURE); 1015 } 1016 1017 return (DDI_SUCCESS); 1018 } 1019 1020 /*ARGSUSED*/ 1021 int 1022 px_lib_msi_getvalid(dev_info_t *dip, msinum_t msi_num, 1023 pci_msi_valid_state_t *msi_valid_state) 1024 { 1025 uint64_t ret; 1026 1027 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: dip 0x%p msi_num 0x%x\n", 1028 dip, msi_num); 1029 1030 if ((ret = hvio_msi_getvalid(DIP_TO_HANDLE(dip), 1031 msi_num, msi_valid_state)) != H_EOK) { 1032 DBG(DBG_LIB_MSI, dip, 1033 "hvio_msi_getvalid failed, ret 0x%lx\n", ret); 1034 return (DDI_FAILURE); 1035 } 1036 1037 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: msiq_id 0x%x\n", 1038 *msi_valid_state); 1039 1040 return (DDI_SUCCESS); 1041 } 1042 1043 /*ARGSUSED*/ 1044 int 1045 px_lib_msi_setvalid(dev_info_t *dip, msinum_t msi_num, 1046 pci_msi_valid_state_t msi_valid_state) 1047 { 1048 uint64_t ret; 1049 1050 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setvalid: dip 0x%p msi_num 0x%x " 1051 "msi_valid_state 0x%x\n", dip, msi_num, msi_valid_state); 1052 1053 if ((ret = hvio_msi_setvalid(DIP_TO_HANDLE(dip), 1054 msi_num, msi_valid_state)) != H_EOK) { 1055 DBG(DBG_LIB_MSI, dip, 1056 "hvio_msi_setvalid failed, ret 0x%lx\n", ret); 1057 return (DDI_FAILURE); 1058 } 1059 1060 return (DDI_SUCCESS); 1061 } 1062 1063 /*ARGSUSED*/ 1064 int 1065 px_lib_msi_getstate(dev_info_t *dip, msinum_t msi_num, 1066 pci_msi_state_t *msi_state) 1067 { 1068 uint64_t ret; 1069 1070 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: dip 0x%p msi_num 0x%x\n", 1071 dip, msi_num); 1072 1073 if ((ret = hvio_msi_getstate(DIP_TO_HANDLE(dip), 1074 msi_num, msi_state)) != H_EOK) { 1075 DBG(DBG_LIB_MSI, dip, 1076 "hvio_msi_getstate failed, ret 0x%lx\n", ret); 1077 return (DDI_FAILURE); 1078 } 1079 1080 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: msi_state 0x%x\n", 1081 *msi_state); 1082 1083 return (DDI_SUCCESS); 1084 } 1085 1086 /*ARGSUSED*/ 1087 int 1088 px_lib_msi_setstate(dev_info_t *dip, msinum_t msi_num, 1089 pci_msi_state_t msi_state) 1090 { 1091 uint64_t ret; 1092 1093 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setstate: dip 0x%p msi_num 0x%x " 1094 "msi_state 0x%x\n", dip, msi_num, msi_state); 1095 1096 if ((ret = hvio_msi_setstate(DIP_TO_HANDLE(dip), 1097 msi_num, msi_state)) != H_EOK) { 1098 DBG(DBG_LIB_MSI, dip, 1099 "hvio_msi_setstate failed, ret 0x%lx\n", ret); 1100 return (DDI_FAILURE); 1101 } 1102 1103 return (DDI_SUCCESS); 1104 } 1105 1106 /* 1107 * MSG Functions: 1108 */ 1109 /*ARGSUSED*/ 1110 int 1111 px_lib_msg_getmsiq(dev_info_t *dip, pcie_msg_type_t msg_type, 1112 msiqid_t *msiq_id) 1113 { 1114 uint64_t ret; 1115 1116 DBG(DBG_LIB_MSG, dip, "px_lib_msg_getmsiq: dip 0x%p msg_type 0x%x\n", 1117 dip, msg_type); 1118 1119 if ((ret = hvio_msg_getmsiq(DIP_TO_HANDLE(dip), 1120 msg_type, msiq_id)) != H_EOK) { 1121 DBG(DBG_LIB_MSG, dip, 1122 "hvio_msg_getmsiq failed, ret 0x%lx\n", ret); 1123 return (DDI_FAILURE); 1124 } 1125 1126 DBG(DBG_LIB_MSI, dip, "px_lib_msg_getmsiq: msiq_id 0x%x\n", 1127 *msiq_id); 1128 1129 return (DDI_SUCCESS); 1130 } 1131 1132 /*ARGSUSED*/ 1133 int 1134 px_lib_msg_setmsiq(dev_info_t *dip, pcie_msg_type_t msg_type, 1135 msiqid_t msiq_id) 1136 { 1137 uint64_t ret; 1138 1139 DBG(DBG_LIB_MSG, dip, "px_lib_msi_setstate: dip 0x%p msg_type 0x%x " 1140 "msiq_id 0x%x\n", dip, msg_type, msiq_id); 1141 1142 if ((ret = hvio_msg_setmsiq(DIP_TO_HANDLE(dip), 1143 msg_type, msiq_id)) != H_EOK) { 1144 DBG(DBG_LIB_MSG, dip, 1145 "hvio_msg_setmsiq failed, ret 0x%lx\n", ret); 1146 return (DDI_FAILURE); 1147 } 1148 1149 return (DDI_SUCCESS); 1150 } 1151 1152 /*ARGSUSED*/ 1153 int 1154 px_lib_msg_getvalid(dev_info_t *dip, pcie_msg_type_t msg_type, 1155 pcie_msg_valid_state_t *msg_valid_state) 1156 { 1157 uint64_t ret; 1158 1159 DBG(DBG_LIB_MSG, dip, "px_lib_msg_getvalid: dip 0x%p msg_type 0x%x\n", 1160 dip, msg_type); 1161 1162 if ((ret = hvio_msg_getvalid(DIP_TO_HANDLE(dip), msg_type, 1163 msg_valid_state)) != H_EOK) { 1164 DBG(DBG_LIB_MSG, dip, 1165 "hvio_msg_getvalid failed, ret 0x%lx\n", ret); 1166 return (DDI_FAILURE); 1167 } 1168 1169 DBG(DBG_LIB_MSI, dip, "px_lib_msg_getvalid: msg_valid_state 0x%x\n", 1170 *msg_valid_state); 1171 1172 return (DDI_SUCCESS); 1173 } 1174 1175 /*ARGSUSED*/ 1176 int 1177 px_lib_msg_setvalid(dev_info_t *dip, pcie_msg_type_t msg_type, 1178 pcie_msg_valid_state_t msg_valid_state) 1179 { 1180 uint64_t ret; 1181 1182 DBG(DBG_LIB_MSG, dip, "px_lib_msg_setvalid: dip 0x%p msg_type 0x%x " 1183 "msg_valid_state 0x%x\n", dip, msg_type, msg_valid_state); 1184 1185 if ((ret = hvio_msg_setvalid(DIP_TO_HANDLE(dip), msg_type, 1186 msg_valid_state)) != H_EOK) { 1187 DBG(DBG_LIB_MSG, dip, 1188 "hvio_msg_setvalid failed, ret 0x%lx\n", ret); 1189 return (DDI_FAILURE); 1190 } 1191 1192 return (DDI_SUCCESS); 1193 } 1194 1195 /* 1196 * Suspend/Resume Functions: 1197 * Currently unsupported by hypervisor 1198 */ 1199 int 1200 px_lib_suspend(dev_info_t *dip) 1201 { 1202 px_t *px_p = DIP_TO_STATE(dip); 1203 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 1204 devhandle_t dev_hdl, xbus_dev_hdl; 1205 uint64_t ret; 1206 1207 DBG(DBG_DETACH, dip, "px_lib_suspend: dip 0x%p\n", dip); 1208 1209 dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_CSR]; 1210 xbus_dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_XBC]; 1211 1212 if ((ret = hvio_suspend(dev_hdl, pxu_p)) == H_EOK) { 1213 px_p->px_cb_p->xbc_attachcnt--; 1214 if (px_p->px_cb_p->xbc_attachcnt == 0) 1215 if ((ret = hvio_cb_suspend(xbus_dev_hdl, pxu_p)) 1216 != H_EOK) 1217 px_p->px_cb_p->xbc_attachcnt++; 1218 } 1219 1220 return ((ret != H_EOK) ? DDI_FAILURE: DDI_SUCCESS); 1221 } 1222 1223 void 1224 px_lib_resume(dev_info_t *dip) 1225 { 1226 px_t *px_p = DIP_TO_STATE(dip); 1227 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 1228 devhandle_t dev_hdl, xbus_dev_hdl; 1229 devino_t pec_ino = px_p->px_inos[PX_INTR_PEC]; 1230 devino_t xbc_ino = px_p->px_inos[PX_INTR_XBC]; 1231 1232 DBG(DBG_ATTACH, dip, "px_lib_resume: dip 0x%p\n", dip); 1233 1234 dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_CSR]; 1235 xbus_dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_XBC]; 1236 1237 px_p->px_cb_p->xbc_attachcnt++; 1238 if (px_p->px_cb_p->xbc_attachcnt == 1) 1239 hvio_cb_resume(dev_hdl, xbus_dev_hdl, xbc_ino, pxu_p); 1240 hvio_resume(dev_hdl, pec_ino, pxu_p); 1241 } 1242 1243 /* 1244 * PCI tool Functions: 1245 * Currently unsupported by hypervisor 1246 */ 1247 /*ARGSUSED*/ 1248 int 1249 px_lib_tools_dev_reg_ops(dev_info_t *dip, void *arg, int cmd, int mode) 1250 { 1251 px_t *px_p = DIP_TO_STATE(dip); 1252 1253 DBG(DBG_TOOLS, dip, "px_lib_tools_dev_reg_ops: dip 0x%p arg 0x%p " 1254 "cmd 0x%x mode 0x%x\n", dip, arg, cmd, mode); 1255 1256 return (px_dev_reg_ops(dip, arg, cmd, mode, px_p)); 1257 } 1258 1259 /*ARGSUSED*/ 1260 int 1261 px_lib_tools_bus_reg_ops(dev_info_t *dip, void *arg, int cmd, int mode) 1262 { 1263 DBG(DBG_TOOLS, dip, "px_lib_tools_bus_reg_ops: dip 0x%p arg 0x%p " 1264 "cmd 0x%x mode 0x%x\n", dip, arg, cmd, mode); 1265 1266 return (px_bus_reg_ops(dip, arg, cmd, mode)); 1267 } 1268 1269 /*ARGSUSED*/ 1270 int 1271 px_lib_tools_intr_admn(dev_info_t *dip, void *arg, int cmd, int mode) 1272 { 1273 px_t *px_p = DIP_TO_STATE(dip); 1274 1275 DBG(DBG_TOOLS, dip, "px_lib_tools_intr_admn: dip 0x%p arg 0x%p " 1276 "cmd 0x%x mode 0x%x\n", dip, arg, cmd, mode); 1277 1278 return (px_intr_admn(dip, arg, cmd, mode, px_p)); 1279 } 1280 1281 /* 1282 * Misc Functions: 1283 * Currently unsupported by hypervisor 1284 */ 1285 uint64_t 1286 px_lib_get_cb(dev_info_t *dip) 1287 { 1288 px_t *px_p = DIP_TO_STATE(dip); 1289 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 1290 1291 return (CSR_XR((caddr_t)pxu_p->px_address[PX_REG_XBC], JBUS_SCRATCH_1)); 1292 } 1293 1294 void 1295 px_lib_set_cb(dev_info_t *dip, uint64_t val) 1296 { 1297 px_t *px_p = DIP_TO_STATE(dip); 1298 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 1299 1300 CSR_XS((caddr_t)pxu_p->px_address[PX_REG_XBC], JBUS_SCRATCH_1, val); 1301 } 1302 1303 /*ARGSUSED*/ 1304 int 1305 px_lib_map_vconfig(dev_info_t *dip, 1306 ddi_map_req_t *mp, pci_config_offset_t off, 1307 pci_regspec_t *rp, caddr_t *addrp) 1308 { 1309 /* 1310 * No special config space access services in this layer. 1311 */ 1312 return (DDI_FAILURE); 1313 } 1314 1315 static void 1316 px_lib_clr_errs(px_t *px_p, px_pec_t *pec_p) 1317 { 1318 dev_info_t *rpdip = px_p->px_dip; 1319 px_cb_t *cb_p = px_p->px_cb_p; 1320 int err = PX_OK, ret; 1321 int acctype = pec_p->pec_safeacc_type; 1322 ddi_fm_error_t derr; 1323 1324 /* Create the derr */ 1325 bzero(&derr, sizeof (ddi_fm_error_t)); 1326 derr.fme_version = DDI_FME_VERSION; 1327 derr.fme_ena = fm_ena_generate(0, FM_ENA_FMT1); 1328 derr.fme_flag = acctype; 1329 1330 if (acctype == DDI_FM_ERR_EXPECTED) { 1331 derr.fme_status = DDI_FM_NONFATAL; 1332 ndi_fm_acc_err_set(pec_p->pec_acc_hdl, &derr); 1333 } 1334 1335 mutex_enter(&cb_p->xbc_fm_mutex); 1336 1337 /* send ereport/handle/clear fire registers */ 1338 err = px_err_handle(px_p, &derr, PX_LIB_CALL, B_TRUE); 1339 1340 /* Check all child devices for errors */ 1341 ret = ndi_fm_handler_dispatch(rpdip, NULL, &derr); 1342 1343 mutex_exit(&cb_p->xbc_fm_mutex); 1344 1345 /* 1346 * PX_FATAL_HW indicates a condition recovered from Fatal-Reset, 1347 * therefore it does not cause panic. 1348 */ 1349 if ((err & (PX_FATAL_GOS | PX_FATAL_SW)) || (ret == DDI_FM_FATAL)) 1350 fm_panic("Fatal System Port Error has occurred\n"); 1351 } 1352 1353 #ifdef DEBUG 1354 int px_peekfault_cnt = 0; 1355 int px_pokefault_cnt = 0; 1356 #endif /* DEBUG */ 1357 1358 /*ARGSUSED*/ 1359 static int 1360 px_lib_do_poke(dev_info_t *dip, dev_info_t *rdip, 1361 peekpoke_ctlops_t *in_args) 1362 { 1363 px_t *px_p = DIP_TO_STATE(dip); 1364 px_pec_t *pec_p = px_p->px_pec_p; 1365 int err = DDI_SUCCESS; 1366 on_trap_data_t otd; 1367 1368 mutex_enter(&pec_p->pec_pokefault_mutex); 1369 pec_p->pec_ontrap_data = &otd; 1370 pec_p->pec_safeacc_type = DDI_FM_ERR_POKE; 1371 1372 /* Set up protected environment. */ 1373 if (!on_trap(&otd, OT_DATA_ACCESS)) { 1374 uintptr_t tramp = otd.ot_trampoline; 1375 1376 otd.ot_trampoline = (uintptr_t)&poke_fault; 1377 err = do_poke(in_args->size, (void *)in_args->dev_addr, 1378 (void *)in_args->host_addr); 1379 otd.ot_trampoline = tramp; 1380 } else 1381 err = DDI_FAILURE; 1382 1383 px_lib_clr_errs(px_p, pec_p); 1384 1385 if (otd.ot_trap & OT_DATA_ACCESS) 1386 err = DDI_FAILURE; 1387 1388 /* Take down protected environment. */ 1389 no_trap(); 1390 1391 pec_p->pec_ontrap_data = NULL; 1392 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1393 mutex_exit(&pec_p->pec_pokefault_mutex); 1394 1395 #ifdef DEBUG 1396 if (err == DDI_FAILURE) 1397 px_pokefault_cnt++; 1398 #endif 1399 return (err); 1400 } 1401 1402 /*ARGSUSED*/ 1403 static int 1404 px_lib_do_caut_put(dev_info_t *dip, dev_info_t *rdip, 1405 peekpoke_ctlops_t *cautacc_ctlops_arg) 1406 { 1407 size_t size = cautacc_ctlops_arg->size; 1408 uintptr_t dev_addr = cautacc_ctlops_arg->dev_addr; 1409 uintptr_t host_addr = cautacc_ctlops_arg->host_addr; 1410 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)cautacc_ctlops_arg->handle; 1411 size_t repcount = cautacc_ctlops_arg->repcount; 1412 uint_t flags = cautacc_ctlops_arg->flags; 1413 1414 px_t *px_p = DIP_TO_STATE(dip); 1415 px_pec_t *pec_p = px_p->px_pec_p; 1416 int err = DDI_SUCCESS; 1417 1418 /* 1419 * Note that i_ndi_busop_access_enter ends up grabbing the pokefault 1420 * mutex. 1421 */ 1422 i_ndi_busop_access_enter(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp); 1423 1424 pec_p->pec_ontrap_data = (on_trap_data_t *)hp->ahi_err->err_ontrap; 1425 pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED; 1426 hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED; 1427 1428 if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) { 1429 for (; repcount; repcount--) { 1430 switch (size) { 1431 1432 case sizeof (uint8_t): 1433 i_ddi_put8(hp, (uint8_t *)dev_addr, 1434 *(uint8_t *)host_addr); 1435 break; 1436 1437 case sizeof (uint16_t): 1438 i_ddi_put16(hp, (uint16_t *)dev_addr, 1439 *(uint16_t *)host_addr); 1440 break; 1441 1442 case sizeof (uint32_t): 1443 i_ddi_put32(hp, (uint32_t *)dev_addr, 1444 *(uint32_t *)host_addr); 1445 break; 1446 1447 case sizeof (uint64_t): 1448 i_ddi_put64(hp, (uint64_t *)dev_addr, 1449 *(uint64_t *)host_addr); 1450 break; 1451 } 1452 1453 host_addr += size; 1454 1455 if (flags == DDI_DEV_AUTOINCR) 1456 dev_addr += size; 1457 1458 px_lib_clr_errs(px_p, pec_p); 1459 1460 if (pec_p->pec_ontrap_data->ot_trap & OT_DATA_ACCESS) { 1461 err = DDI_FAILURE; 1462 #ifdef DEBUG 1463 px_pokefault_cnt++; 1464 #endif 1465 break; 1466 } 1467 } 1468 } 1469 1470 i_ddi_notrap((ddi_acc_handle_t)hp); 1471 pec_p->pec_ontrap_data = NULL; 1472 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1473 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp); 1474 hp->ahi_err->err_expected = DDI_FM_ERR_UNEXPECTED; 1475 1476 return (err); 1477 } 1478 1479 1480 int 1481 px_lib_ctlops_poke(dev_info_t *dip, dev_info_t *rdip, 1482 peekpoke_ctlops_t *in_args) 1483 { 1484 return (in_args->handle ? px_lib_do_caut_put(dip, rdip, in_args) : 1485 px_lib_do_poke(dip, rdip, in_args)); 1486 } 1487 1488 1489 /*ARGSUSED*/ 1490 static int 1491 px_lib_do_peek(dev_info_t *dip, peekpoke_ctlops_t *in_args) 1492 { 1493 px_t *px_p = DIP_TO_STATE(dip); 1494 px_pec_t *pec_p = px_p->px_pec_p; 1495 int err = DDI_SUCCESS; 1496 on_trap_data_t otd; 1497 1498 mutex_enter(&pec_p->pec_pokefault_mutex); 1499 pec_p->pec_safeacc_type = DDI_FM_ERR_PEEK; 1500 1501 if (!on_trap(&otd, OT_DATA_ACCESS)) { 1502 uintptr_t tramp = otd.ot_trampoline; 1503 1504 otd.ot_trampoline = (uintptr_t)&peek_fault; 1505 err = do_peek(in_args->size, (void *)in_args->dev_addr, 1506 (void *)in_args->host_addr); 1507 otd.ot_trampoline = tramp; 1508 } else 1509 err = DDI_FAILURE; 1510 1511 no_trap(); 1512 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1513 mutex_exit(&pec_p->pec_pokefault_mutex); 1514 1515 #ifdef DEBUG 1516 if (err == DDI_FAILURE) 1517 px_peekfault_cnt++; 1518 #endif 1519 return (err); 1520 } 1521 1522 1523 static int 1524 px_lib_do_caut_get(dev_info_t *dip, peekpoke_ctlops_t *cautacc_ctlops_arg) 1525 { 1526 size_t size = cautacc_ctlops_arg->size; 1527 uintptr_t dev_addr = cautacc_ctlops_arg->dev_addr; 1528 uintptr_t host_addr = cautacc_ctlops_arg->host_addr; 1529 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)cautacc_ctlops_arg->handle; 1530 size_t repcount = cautacc_ctlops_arg->repcount; 1531 uint_t flags = cautacc_ctlops_arg->flags; 1532 1533 px_t *px_p = DIP_TO_STATE(dip); 1534 px_pec_t *pec_p = px_p->px_pec_p; 1535 int err = DDI_SUCCESS; 1536 1537 /* 1538 * Note that i_ndi_busop_access_enter ends up grabbing the pokefault 1539 * mutex. 1540 */ 1541 i_ndi_busop_access_enter(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp); 1542 1543 pec_p->pec_ontrap_data = (on_trap_data_t *)hp->ahi_err->err_ontrap; 1544 pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED; 1545 hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED; 1546 1547 if (repcount == 1) { 1548 if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) { 1549 i_ddi_caut_get(size, (void *)dev_addr, 1550 (void *)host_addr); 1551 } else { 1552 int i; 1553 uint8_t *ff_addr = (uint8_t *)host_addr; 1554 for (i = 0; i < size; i++) 1555 *ff_addr++ = 0xff; 1556 1557 err = DDI_FAILURE; 1558 #ifdef DEBUG 1559 px_peekfault_cnt++; 1560 #endif 1561 } 1562 } else { 1563 if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) { 1564 for (; repcount; repcount--) { 1565 i_ddi_caut_get(size, (void *)dev_addr, 1566 (void *)host_addr); 1567 1568 host_addr += size; 1569 1570 if (flags == DDI_DEV_AUTOINCR) 1571 dev_addr += size; 1572 } 1573 } else { 1574 err = DDI_FAILURE; 1575 #ifdef DEBUG 1576 px_peekfault_cnt++; 1577 #endif 1578 } 1579 } 1580 1581 i_ddi_notrap((ddi_acc_handle_t)hp); 1582 pec_p->pec_ontrap_data = NULL; 1583 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1584 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp); 1585 hp->ahi_err->err_expected = DDI_FM_ERR_UNEXPECTED; 1586 1587 return (err); 1588 } 1589 1590 /*ARGSUSED*/ 1591 int 1592 px_lib_ctlops_peek(dev_info_t *dip, dev_info_t *rdip, 1593 peekpoke_ctlops_t *in_args, void *result) 1594 { 1595 result = (void *)in_args->host_addr; 1596 return (in_args->handle ? px_lib_do_caut_get(dip, in_args) : 1597 px_lib_do_peek(dip, in_args)); 1598 } 1599 1600 /* 1601 * implements PPM interface 1602 */ 1603 int 1604 px_lib_pmctl(int cmd, px_t *px_p) 1605 { 1606 ASSERT((cmd & ~PPMREQ_MASK) == PPMREQ); 1607 switch (cmd) { 1608 case PPMREQ_PRE_PWR_OFF: 1609 /* 1610 * Currently there is no device power management for 1611 * the root complex (fire). When there is we need to make 1612 * sure that it is at full power before trying to send the 1613 * PME_Turn_Off message. 1614 */ 1615 DBG(DBG_PWR, px_p->px_dip, 1616 "ioctl: request to send PME_Turn_Off\n"); 1617 return (px_goto_l23ready(px_p)); 1618 1619 case PPMREQ_PRE_PWR_ON: 1620 DBG(DBG_PWR, px_p->px_dip, "ioctl: PRE_PWR_ON request\n"); 1621 return (px_pre_pwron_check(px_p)); 1622 1623 case PPMREQ_POST_PWR_ON: 1624 DBG(DBG_PWR, px_p->px_dip, "ioctl: POST_PWR_ON request\n"); 1625 return (px_goto_l0(px_p)); 1626 1627 default: 1628 return (DDI_FAILURE); 1629 } 1630 } 1631 1632 #define MSEC_TO_USEC 1000 1633 1634 /* 1635 * sends PME_Turn_Off message to put the link in L2/L3 ready state. 1636 * called by px_ioctl. 1637 * returns DDI_SUCCESS or DDI_FAILURE 1638 * 1. Wait for link to be in L1 state (link status reg) 1639 * 2. write to PME_Turn_off reg to boradcast 1640 * 3. set timeout 1641 * 4. If timeout, return failure. 1642 * 5. If PM_TO_Ack, wait till link is in L2/L3 ready 1643 */ 1644 static int 1645 px_goto_l23ready(px_t *px_p) 1646 { 1647 pcie_pwr_t *pwr_p; 1648 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 1649 caddr_t csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR]; 1650 int ret = DDI_SUCCESS; 1651 clock_t end, timeleft; 1652 int mutex_held = 1; 1653 1654 /* If no PM info, return failure */ 1655 if (!PCIE_PMINFO(px_p->px_dip) || 1656 !(pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip))) 1657 return (DDI_FAILURE); 1658 1659 mutex_enter(&pwr_p->pwr_lock); 1660 mutex_enter(&px_p->px_l23ready_lock); 1661 /* Clear the PME_To_ACK receieved flag */ 1662 px_p->px_pm_flags &= ~PX_PMETOACK_RECVD; 1663 /* 1664 * When P25 is the downstream device, after receiving 1665 * PME_To_ACK, fire will go to Detect state, which causes 1666 * the link down event. Inform FMA that this is expected. 1667 * In case of all other cards complaint with the pci express 1668 * spec, this will happen when the power is re-applied. FMA 1669 * code will clear this flag after one instance of LDN. Since 1670 * there will not be a LDN event for the spec compliant cards, 1671 * we need to clear the flag after receiving PME_To_ACK. 1672 */ 1673 px_p->px_pm_flags |= PX_LDN_EXPECTED; 1674 if (px_send_pme_turnoff(csr_base) != DDI_SUCCESS) { 1675 ret = DDI_FAILURE; 1676 goto l23ready_done; 1677 } 1678 px_p->px_pm_flags |= PX_PME_TURNOFF_PENDING; 1679 1680 end = ddi_get_lbolt() + drv_usectohz(px_pme_to_ack_timeout); 1681 while (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) { 1682 timeleft = cv_timedwait(&px_p->px_l23ready_cv, 1683 &px_p->px_l23ready_lock, end); 1684 /* 1685 * if cv_timedwait returns -1, it is either 1686 * 1) timed out or 1687 * 2) there was a pre-mature wakeup but by the time 1688 * cv_timedwait is called again end < lbolt i.e. 1689 * end is in the past. 1690 * 3) By the time we make first cv_timedwait call, 1691 * end < lbolt is true. 1692 */ 1693 if (timeleft == -1) 1694 break; 1695 } 1696 if (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) { 1697 /* 1698 * Either timedout or interrupt didn't get a 1699 * chance to grab the mutex and set the flag. 1700 * release the mutex and delay for sometime. 1701 * This will 1) give a chance for interrupt to 1702 * set the flag 2) creates a delay between two 1703 * consequetive requests. 1704 */ 1705 mutex_exit(&px_p->px_l23ready_lock); 1706 delay(drv_usectohz(50 * MSEC_TO_USEC)); 1707 mutex_held = 0; 1708 if (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) { 1709 ret = DDI_FAILURE; 1710 DBG(DBG_PWR, px_p->px_dip, " Timed out while waiting" 1711 " for PME_TO_ACK\n"); 1712 } 1713 } 1714 px_p->px_pm_flags &= 1715 ~(PX_PME_TURNOFF_PENDING | PX_PMETOACK_RECVD | PX_LDN_EXPECTED); 1716 1717 l23ready_done: 1718 if (mutex_held) 1719 mutex_exit(&px_p->px_l23ready_lock); 1720 /* 1721 * Wait till link is in L1 idle, if sending PME_Turn_Off 1722 * was succesful. 1723 */ 1724 if (ret == DDI_SUCCESS) { 1725 if (px_link_wait4l1idle(csr_base) != DDI_SUCCESS) { 1726 DBG(DBG_PWR, px_p->px_dip, " Link is not at L1" 1727 " even though we received PME_To_ACK.\n"); 1728 /* 1729 * Workaround for hardware bug with P25. 1730 * Due to a hardware bug with P25, link state 1731 * will be Detect state rather than L1 after 1732 * link is transitioned to L23Ready state. Since 1733 * we don't know whether link is L23ready state 1734 * without Fire's state being L1_idle, we delay 1735 * here just to make sure that we wait till link 1736 * is transitioned to L23Ready state. 1737 */ 1738 delay(drv_usectohz(100 * MSEC_TO_USEC)); 1739 } 1740 pwr_p->pwr_link_lvl = PM_LEVEL_L3; 1741 1742 } 1743 mutex_exit(&pwr_p->pwr_lock); 1744 return (ret); 1745 } 1746 1747 /* 1748 * Message interrupt handler intended to be shared for both 1749 * PME and PME_TO_ACK msg handling, currently only handles 1750 * PME_To_ACK message. 1751 */ 1752 uint_t 1753 px_pmeq_intr(caddr_t arg) 1754 { 1755 px_t *px_p = (px_t *)arg; 1756 1757 DBG(DBG_PWR, px_p->px_dip, " PME_To_ACK received \n"); 1758 mutex_enter(&px_p->px_l23ready_lock); 1759 cv_broadcast(&px_p->px_l23ready_cv); 1760 if (px_p->px_pm_flags & PX_PME_TURNOFF_PENDING) { 1761 px_p->px_pm_flags |= PX_PMETOACK_RECVD; 1762 } else { 1763 /* 1764 * This maybe the second ack received. If so then, 1765 * we should be receiving it during wait4L1 stage. 1766 */ 1767 px_p->px_pmetoack_ignored++; 1768 } 1769 mutex_exit(&px_p->px_l23ready_lock); 1770 return (DDI_INTR_CLAIMED); 1771 } 1772 1773 static int 1774 px_pre_pwron_check(px_t *px_p) 1775 { 1776 pcie_pwr_t *pwr_p; 1777 1778 /* If no PM info, return failure */ 1779 if (!PCIE_PMINFO(px_p->px_dip) || 1780 !(pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip))) 1781 return (DDI_FAILURE); 1782 1783 /* 1784 * For the spec compliant downstream cards link down 1785 * is expected when the device is powered on. 1786 */ 1787 px_p->px_pm_flags |= PX_LDN_EXPECTED; 1788 return (pwr_p->pwr_link_lvl == PM_LEVEL_L3 ? DDI_SUCCESS : DDI_FAILURE); 1789 } 1790 1791 static int 1792 px_goto_l0(px_t *px_p) 1793 { 1794 pcie_pwr_t *pwr_p; 1795 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 1796 caddr_t csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR]; 1797 int ret = DDI_SUCCESS; 1798 clock_t end, timeleft; 1799 int mutex_held = 1; 1800 1801 /* If no PM info, return failure */ 1802 if (!PCIE_PMINFO(px_p->px_dip) || 1803 !(pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip))) 1804 return (DDI_FAILURE); 1805 1806 mutex_enter(&pwr_p->pwr_lock); 1807 mutex_enter(&px_p->px_lupsoft_lock); 1808 /* Clear the LINKUP_RECVD receieved flag */ 1809 px_p->px_pm_flags &= ~PX_LINKUP_RECVD; 1810 /* 1811 * Set flags LUP_EXPECTED to inform FMA code that LUP is 1812 * expected as part of link training and no ereports should 1813 * be posted for this event. FMA code will clear this flag 1814 * after one instance of this event. In case of P25, there 1815 * will not be a LDN event. So clear the flag set at PRE_PWRON 1816 * time. 1817 */ 1818 px_p->px_pm_flags |= PX_LUP_EXPECTED; 1819 px_p->px_pm_flags &= ~PX_LDN_EXPECTED; 1820 if (px_link_retrain(csr_base) != DDI_SUCCESS) { 1821 ret = DDI_FAILURE; 1822 goto l0_done; 1823 } 1824 px_p->px_pm_flags |= PX_LINKUP_PENDING; 1825 1826 end = ddi_get_lbolt() + drv_usectohz(px_linkup_timeout); 1827 while (!(px_p->px_pm_flags & PX_LINKUP_RECVD)) { 1828 timeleft = cv_timedwait(&px_p->px_lup_cv, 1829 &px_p->px_lupsoft_lock, end); 1830 /* 1831 * if cv_timedwait returns -1, it is either 1832 * 1) timed out or 1833 * 2) there was a pre-mature wakeup but by the time 1834 * cv_timedwait is called again end < lbolt i.e. 1835 * end is in the past. 1836 * 3) By the time we make first cv_timedwait call, 1837 * end < lbolt is true. 1838 */ 1839 if (timeleft == -1) 1840 break; 1841 } 1842 if (!(px_p->px_pm_flags & PX_LINKUP_RECVD)) { 1843 /* 1844 * Either timedout or interrupt didn't get a 1845 * chance to grab the mutex and set the flag. 1846 * release the mutex and delay for sometime. 1847 * This will 1) give a chance for interrupt to 1848 * set the flag 2) creates a delay between two 1849 * consequetive requests. 1850 */ 1851 mutex_exit(&px_p->px_lupsoft_lock); 1852 mutex_held = 0; 1853 delay(drv_usectohz(50 * MSEC_TO_USEC)); 1854 if (!(px_p->px_pm_flags & PX_LINKUP_RECVD)) { 1855 ret = DDI_FAILURE; 1856 DBG(DBG_PWR, px_p->px_dip, " Timed out while waiting" 1857 " for link up\n"); 1858 } 1859 } 1860 px_p->px_pm_flags &= 1861 ~(PX_LINKUP_PENDING | PX_LINKUP_RECVD | PX_LUP_EXPECTED); 1862 1863 l0_done: 1864 if (mutex_held) 1865 mutex_exit(&px_p->px_lupsoft_lock); 1866 px_enable_detect_quiet(csr_base); 1867 if (ret == DDI_SUCCESS) 1868 pwr_p->pwr_link_lvl = PM_LEVEL_L0; 1869 mutex_exit(&pwr_p->pwr_lock); 1870 return (ret); 1871 } 1872 1873 uint_t 1874 px_lup_softintr(caddr_t arg) 1875 { 1876 px_t *px_p = (px_t *)arg; 1877 1878 DBG(DBG_PWR, px_p->px_dip, " Link up soft interrupt received \n"); 1879 mutex_enter(&px_p->px_lup_lock); 1880 if (!(px_p->px_lupsoft_pending > 0)) { 1881 /* Spurious */ 1882 mutex_exit(&px_p->px_lup_lock); 1883 return (DDI_INTR_UNCLAIMED); 1884 } 1885 px_p->px_lupsoft_pending--; 1886 if (px_p->px_lupsoft_pending > 0) { 1887 /* More than one lup soft intr posted - unlikely */ 1888 mutex_exit(&px_p->px_lup_lock); 1889 return (DDI_INTR_UNCLAIMED); 1890 } 1891 mutex_exit(&px_p->px_lup_lock); 1892 1893 mutex_enter(&px_p->px_lupsoft_lock); 1894 cv_broadcast(&px_p->px_lup_cv); 1895 if (px_p->px_pm_flags & PX_LINKUP_PENDING) { 1896 px_p->px_pm_flags |= PX_LINKUP_RECVD; 1897 } else { 1898 /* Nobody waiting for this! */ 1899 px_p->px_lup_ignored++; 1900 } 1901 mutex_exit(&px_p->px_lupsoft_lock); 1902 return (DDI_INTR_CLAIMED); 1903 } 1904 1905 /* 1906 * Extract the drivers binding name to identify which chip we're binding to. 1907 * Whenever a new bus bridge is created, the driver alias entry should be 1908 * added here to identify the device if needed. If a device isn't added, 1909 * the identity defaults to PX_CHIP_UNIDENTIFIED. 1910 */ 1911 static uint32_t 1912 px_identity_chip(px_t *px_p) 1913 { 1914 dev_info_t *dip = px_p->px_dip; 1915 char *name = ddi_binding_name(dip); 1916 uint32_t revision = 0; 1917 1918 revision = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1919 "module-revision#", 0); 1920 1921 /* Check for Fire driver binding name */ 1922 if ((strcmp(name, "pci108e,80f0") == 0) || 1923 (strcmp(name, "pciex108e,80f0") == 0)) { 1924 DBG(DBG_ATTACH, dip, "px_identity_chip: %s%d: " 1925 "name %s module-revision %d\n", ddi_driver_name(dip), 1926 ddi_get_instance(dip), name, revision); 1927 1928 return (PX_CHIP_ID(PX_CHIP_FIRE, revision, 0x00)); 1929 } 1930 1931 DBG(DBG_ATTACH, dip, "%s%d: Unknown PCI Express Host bridge %s %x\n", 1932 ddi_driver_name(dip), ddi_get_instance(dip), name, revision); 1933 1934 return (PX_CHIP_UNIDENTIFIED); 1935 } 1936 1937 int 1938 px_err_add_intr(px_fault_t *px_fault_p) 1939 { 1940 dev_info_t *dip = px_fault_p->px_fh_dip; 1941 px_t *px_p = DIP_TO_STATE(dip); 1942 1943 VERIFY(add_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL, 1944 px_fault_p->px_err_func, (caddr_t)px_fault_p, NULL) == 0); 1945 1946 px_ib_intr_enable(px_p, intr_dist_cpuid(), px_fault_p->px_intr_ino); 1947 1948 return (DDI_SUCCESS); 1949 } 1950 1951 void 1952 px_err_rem_intr(px_fault_t *px_fault_p) 1953 { 1954 dev_info_t *dip = px_fault_p->px_fh_dip; 1955 px_t *px_p = DIP_TO_STATE(dip); 1956 1957 rem_ivintr(px_fault_p->px_fh_sysino, NULL); 1958 1959 px_ib_intr_disable(px_p->px_ib_p, px_fault_p->px_intr_ino, 1960 IB_INTR_WAIT); 1961 } 1962 1963 #ifdef FMA 1964 void 1965 px_fill_rc_status(px_fault_t *px_fault_p, pciex_rc_error_regs_t *rc_status) 1966 { 1967 /* populate the rc_status by reading the registers - TBD */ 1968 } 1969 #endif /* FMA */ 1970 1971 /* 1972 * Unprotected raw reads/writes of fabric device's config space. 1973 * Only used for temporary PCI-E Fabric Error Handling. 1974 */ 1975 uint32_t 1976 px_fab_get(px_t *px_p, pcie_req_id_t bdf, uint16_t offset) { 1977 px_ranges_t *rp = px_p->px_ranges_p; 1978 uint64_t range_prop, base_addr; 1979 int bank = PCI_REG_ADDR_G(PCI_ADDR_CONFIG); 1980 uint32_t val; 1981 1982 /* Get Fire's Physical Base Address */ 1983 range_prop = (((uint64_t)(rp[bank].parent_high & 0x7ff)) << 32) | 1984 rp[bank].parent_low; 1985 1986 /* Get config space first. */ 1987 base_addr = range_prop + PX_BDF_TO_CFGADDR(bdf, offset); 1988 1989 val = ldphysio(base_addr); 1990 1991 return (LE_32(val)); 1992 } 1993 1994 void 1995 px_fab_set(px_t *px_p, pcie_req_id_t bdf, uint16_t offset, 1996 uint32_t val) { 1997 px_ranges_t *rp = px_p->px_ranges_p; 1998 uint64_t range_prop, base_addr; 1999 int bank = PCI_REG_ADDR_G(PCI_ADDR_CONFIG); 2000 2001 /* Get Fire's Physical Base Address */ 2002 range_prop = (((uint64_t)(rp[bank].parent_high & 0x7ff)) << 32) | 2003 rp[bank].parent_low; 2004 2005 /* Get config space first. */ 2006 base_addr = range_prop + PX_BDF_TO_CFGADDR(bdf, offset); 2007 2008 stphysio(base_addr, LE_32(val)); 2009 } 2010 2011 /* 2012 * cpr callback 2013 * 2014 * disable fabric error msg interrupt prior to suspending 2015 * all device drivers; re-enable fabric error msg interrupt 2016 * after all devices are resumed. 2017 */ 2018 static boolean_t 2019 px_cpr_callb(void *arg, int code) 2020 { 2021 px_t *px_p = (px_t *)arg; 2022 px_ib_t *ib_p = px_p->px_ib_p; 2023 px_pec_t *pec_p = px_p->px_pec_p; 2024 pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p; 2025 caddr_t csr_base; 2026 devino_t ce_ino, nf_ino, f_ino; 2027 px_ib_ino_info_t *ce_ino_p, *nf_ino_p, *f_ino_p; 2028 uint64_t imu_log_enable, imu_intr_enable; 2029 uint64_t imu_log_mask, imu_intr_mask; 2030 2031 ce_ino = px_msiqid_to_devino(px_p, pec_p->pec_corr_msg_msiq_id); 2032 nf_ino = px_msiqid_to_devino(px_p, pec_p->pec_non_fatal_msg_msiq_id); 2033 f_ino = px_msiqid_to_devino(px_p, pec_p->pec_fatal_msg_msiq_id); 2034 csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR]; 2035 2036 imu_log_enable = CSR_XR(csr_base, IMU_ERROR_LOG_ENABLE); 2037 imu_intr_enable = CSR_XR(csr_base, IMU_INTERRUPT_ENABLE); 2038 2039 imu_log_mask = BITMASK(IMU_ERROR_LOG_ENABLE_FATAL_MES_NOT_EN_LOG_EN) | 2040 BITMASK(IMU_ERROR_LOG_ENABLE_NONFATAL_MES_NOT_EN_LOG_EN) | 2041 BITMASK(IMU_ERROR_LOG_ENABLE_COR_MES_NOT_EN_LOG_EN); 2042 2043 imu_intr_mask = 2044 BITMASK(IMU_INTERRUPT_ENABLE_FATAL_MES_NOT_EN_S_INT_EN) | 2045 BITMASK(IMU_INTERRUPT_ENABLE_NONFATAL_MES_NOT_EN_S_INT_EN) | 2046 BITMASK(IMU_INTERRUPT_ENABLE_COR_MES_NOT_EN_S_INT_EN) | 2047 BITMASK(IMU_INTERRUPT_ENABLE_FATAL_MES_NOT_EN_P_INT_EN) | 2048 BITMASK(IMU_INTERRUPT_ENABLE_NONFATAL_MES_NOT_EN_P_INT_EN) | 2049 BITMASK(IMU_INTERRUPT_ENABLE_COR_MES_NOT_EN_P_INT_EN); 2050 2051 switch (code) { 2052 case CB_CODE_CPR_CHKPT: 2053 /* disable imu rbne on corr/nonfatal/fatal errors */ 2054 CSR_XS(csr_base, IMU_ERROR_LOG_ENABLE, 2055 imu_log_enable & (~imu_log_mask)); 2056 2057 CSR_XS(csr_base, IMU_INTERRUPT_ENABLE, 2058 imu_intr_enable & (~imu_intr_mask)); 2059 2060 /* disable CORR intr mapping */ 2061 px_ib_intr_disable(ib_p, ce_ino, IB_INTR_NOWAIT); 2062 2063 /* disable NON FATAL intr mapping */ 2064 px_ib_intr_disable(ib_p, nf_ino, IB_INTR_NOWAIT); 2065 2066 /* disable FATAL intr mapping */ 2067 px_ib_intr_disable(ib_p, f_ino, IB_INTR_NOWAIT); 2068 2069 break; 2070 2071 case CB_CODE_CPR_RESUME: 2072 mutex_enter(&ib_p->ib_ino_lst_mutex); 2073 2074 ce_ino_p = px_ib_locate_ino(ib_p, ce_ino); 2075 nf_ino_p = px_ib_locate_ino(ib_p, nf_ino); 2076 f_ino_p = px_ib_locate_ino(ib_p, f_ino); 2077 2078 /* enable CORR intr mapping */ 2079 if (ce_ino_p) 2080 px_ib_intr_enable(px_p, ce_ino_p->ino_cpuid, ce_ino); 2081 else 2082 cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to " 2083 "reenable PCIe Correctable msg intr.\n"); 2084 2085 /* enable NON FATAL intr mapping */ 2086 if (nf_ino_p) 2087 px_ib_intr_enable(px_p, nf_ino_p->ino_cpuid, nf_ino); 2088 else 2089 cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to " 2090 "reenable PCIe Non Fatal msg intr.\n"); 2091 2092 /* enable FATAL intr mapping */ 2093 if (f_ino_p) 2094 px_ib_intr_enable(px_p, f_ino_p->ino_cpuid, f_ino); 2095 else 2096 cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to " 2097 "reenable PCIe Fatal msg intr.\n"); 2098 2099 mutex_exit(&ib_p->ib_ino_lst_mutex); 2100 2101 /* enable corr/nonfatal/fatal not enable error */ 2102 CSR_XS(csr_base, IMU_ERROR_LOG_ENABLE, (imu_log_enable | 2103 (imu_log_mask & px_imu_log_mask))); 2104 CSR_XS(csr_base, IMU_INTERRUPT_ENABLE, (imu_intr_enable | 2105 (imu_intr_mask & px_imu_intr_mask))); 2106 2107 break; 2108 } 2109 2110 return (B_TRUE); 2111 } 2112 2113 /* 2114 * add cpr callback 2115 */ 2116 void 2117 px_cpr_add_callb(px_t *px_p) 2118 { 2119 px_p->px_cprcb_id = callb_add(px_cpr_callb, (void *)px_p, 2120 CB_CL_CPR_POST_USER, "px_cpr"); 2121 } 2122 2123 /* 2124 * remove cpr callback 2125 */ 2126 void 2127 px_cpr_rem_callb(px_t *px_p) 2128 { 2129 (void) callb_delete(px_p->px_cprcb_id); 2130 } 2131