1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/sysmacros.h> 28 #include <sys/ddi.h> 29 #include <sys/async.h> 30 #include <sys/sunddi.h> 31 #include <sys/ddifm.h> 32 #include <sys/fm/protocol.h> 33 #include <sys/vmem.h> 34 #include <sys/intr.h> 35 #include <sys/ivintr.h> 36 #include <sys/errno.h> 37 #include <sys/hypervisor_api.h> 38 #include <sys/hsvc.h> 39 #include <px_obj.h> 40 #include <sys/machsystm.h> 41 #include <sys/hotplug/pci/pcihp.h> 42 #include "px_lib4v.h" 43 #include "px_err.h" 44 #include <vm/vm_dep.h> 45 #include <vm/hat_sfmmu.h> 46 47 /* mask for the ranges property in calculating the real PFN range */ 48 uint_t px_ranges_phi_mask = ((1 << 28) -1); 49 50 /* 51 * Hypervisor VPCI services information for the px nexus driver. 52 */ 53 static uint64_t px_vpci_min_ver; /* Negotiated VPCI API minor version */ 54 static uint_t px_vpci_users = 0; /* VPCI API users */ 55 56 static hsvc_info_t px_hsvc = { 57 HSVC_REV_1, NULL, HSVC_GROUP_VPCI, PX_VPCI_MAJOR_VER, 58 PX_VPCI_MINOR_VER, "PX" 59 }; 60 61 int 62 px_lib_dev_init(dev_info_t *dip, devhandle_t *dev_hdl) 63 { 64 px_nexus_regspec_t *rp; 65 uint_t reglen; 66 int ret; 67 68 uint64_t mjrnum; 69 uint64_t mnrnum; 70 71 DBG(DBG_ATTACH, dip, "px_lib_dev_init: dip 0x%p\n", dip); 72 73 /* 74 * Check HV intr group api versioning. 75 * This driver uses the old interrupt routines which are supported 76 * in old firmware in the CORE API group and in newer firmware in 77 * the INTR API group. Support for these calls will be dropped 78 * once the INTR API group major goes to 2. 79 */ 80 if ((hsvc_version(HSVC_GROUP_INTR, &mjrnum, &mnrnum) == 0) && 81 (mjrnum > 1)) { 82 cmn_err(CE_WARN, "niumx: unsupported intr api group: " 83 "maj:0x%lx, min:0x%lx", mjrnum, mnrnum); 84 return (ENOTSUP); 85 } 86 87 ret = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 88 "reg", (uchar_t **)&rp, ®len); 89 if (ret != DDI_PROP_SUCCESS) { 90 DBG(DBG_ATTACH, dip, "px_lib_dev_init failed ret=%d\n", ret); 91 return (DDI_FAILURE); 92 } 93 94 /* 95 * Initilize device handle. The device handle uniquely identifies 96 * a SUN4V device. It consists of the lower 28-bits of the hi-cell 97 * of the first entry of the SUN4V device's "reg" property as 98 * defined by the SUN4V Bus Binding to Open Firmware. 99 */ 100 *dev_hdl = (devhandle_t)((rp->phys_addr >> 32) & DEVHDLE_MASK); 101 ddi_prop_free(rp); 102 103 /* 104 * hotplug implementation requires this property to be associated with 105 * any indirect PCI config access services 106 */ 107 (void) ddi_prop_update_int(makedevice(ddi_driver_major(dip), 108 PCIHP_AP_MINOR_NUM(ddi_get_instance(dip), PCIHP_DEVCTL_MINOR)), dip, 109 PCI_BUS_CONF_MAP_PROP, 1); 110 111 DBG(DBG_ATTACH, dip, "px_lib_dev_init: dev_hdl 0x%llx\n", *dev_hdl); 112 113 /* 114 * Negotiate the API version for VPCI hypervisor services. 115 */ 116 if (px_vpci_users++) 117 return (DDI_SUCCESS); 118 119 if ((ret = hsvc_register(&px_hsvc, &px_vpci_min_ver)) != 0) { 120 cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services " 121 "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d\n", 122 px_hsvc.hsvc_modname, px_hsvc.hsvc_group, 123 px_hsvc.hsvc_major, px_hsvc.hsvc_minor, ret); 124 125 return (DDI_FAILURE); 126 } 127 128 DBG(DBG_ATTACH, dip, "px_lib_dev_init: negotiated VPCI API version, " 129 "major 0x%lx minor 0x%lx\n", px_hsvc.hsvc_major, px_vpci_min_ver); 130 131 return (DDI_SUCCESS); 132 } 133 134 /*ARGSUSED*/ 135 int 136 px_lib_dev_fini(dev_info_t *dip) 137 { 138 DBG(DBG_DETACH, dip, "px_lib_dev_fini: dip 0x%p\n", dip); 139 140 (void) ddi_prop_remove(makedevice(ddi_driver_major(dip), 141 PCIHP_AP_MINOR_NUM(ddi_get_instance(dip), PCIHP_DEVCTL_MINOR)), dip, 142 PCI_BUS_CONF_MAP_PROP); 143 144 if (--px_vpci_users == 0) 145 (void) hsvc_unregister(&px_hsvc); 146 147 return (DDI_SUCCESS); 148 } 149 150 /*ARGSUSED*/ 151 int 152 px_lib_intr_devino_to_sysino(dev_info_t *dip, devino_t devino, 153 sysino_t *sysino) 154 { 155 uint64_t ret; 156 157 DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: dip 0x%p " 158 "devino 0x%x\n", dip, devino); 159 160 if ((ret = hvio_intr_devino_to_sysino(DIP_TO_HANDLE(dip), 161 devino, sysino)) != H_EOK) { 162 DBG(DBG_LIB_INT, dip, 163 "hvio_intr_devino_to_sysino failed, ret 0x%lx\n", ret); 164 return (DDI_FAILURE); 165 } 166 167 DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: sysino 0x%llx\n", 168 *sysino); 169 170 return (DDI_SUCCESS); 171 } 172 173 /*ARGSUSED*/ 174 int 175 px_lib_intr_getvalid(dev_info_t *dip, sysino_t sysino, 176 intr_valid_state_t *intr_valid_state) 177 { 178 uint64_t ret; 179 180 DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: dip 0x%p sysino 0x%llx\n", 181 dip, sysino); 182 183 if ((ret = hvio_intr_getvalid(sysino, 184 (int *)intr_valid_state)) != H_EOK) { 185 DBG(DBG_LIB_INT, dip, "hvio_intr_getvalid failed, ret 0x%lx\n", 186 ret); 187 return (DDI_FAILURE); 188 } 189 190 DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: intr_valid_state 0x%x\n", 191 *intr_valid_state); 192 193 return (DDI_SUCCESS); 194 } 195 196 /*ARGSUSED*/ 197 int 198 px_lib_intr_setvalid(dev_info_t *dip, sysino_t sysino, 199 intr_valid_state_t intr_valid_state) 200 { 201 uint64_t ret; 202 203 DBG(DBG_LIB_INT, dip, "px_lib_intr_setvalid: dip 0x%p sysino 0x%llx " 204 "intr_valid_state 0x%x\n", dip, sysino, intr_valid_state); 205 206 if ((ret = hvio_intr_setvalid(sysino, intr_valid_state)) != H_EOK) { 207 DBG(DBG_LIB_INT, dip, "hvio_intr_setvalid failed, ret 0x%lx\n", 208 ret); 209 return (DDI_FAILURE); 210 } 211 212 return (DDI_SUCCESS); 213 } 214 215 /*ARGSUSED*/ 216 int 217 px_lib_intr_getstate(dev_info_t *dip, sysino_t sysino, 218 intr_state_t *intr_state) 219 { 220 uint64_t ret; 221 222 DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: dip 0x%p sysino 0x%llx\n", 223 dip, sysino); 224 225 if ((ret = hvio_intr_getstate(sysino, (int *)intr_state)) != H_EOK) { 226 DBG(DBG_LIB_INT, dip, "hvio_intr_getstate failed, ret 0x%lx\n", 227 ret); 228 return (DDI_FAILURE); 229 } 230 231 DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: intr_state 0x%x\n", 232 *intr_state); 233 234 return (DDI_SUCCESS); 235 } 236 237 /*ARGSUSED*/ 238 int 239 px_lib_intr_setstate(dev_info_t *dip, sysino_t sysino, 240 intr_state_t intr_state) 241 { 242 uint64_t ret; 243 244 DBG(DBG_LIB_INT, dip, "px_lib_intr_setstate: dip 0x%p sysino 0x%llx " 245 "intr_state 0x%x\n", dip, sysino, intr_state); 246 247 if ((ret = hvio_intr_setstate(sysino, intr_state)) != H_EOK) { 248 DBG(DBG_LIB_INT, dip, "hvio_intr_setstate failed, ret 0x%lx\n", 249 ret); 250 return (DDI_FAILURE); 251 } 252 253 return (DDI_SUCCESS); 254 } 255 256 /*ARGSUSED*/ 257 int 258 px_lib_intr_gettarget(dev_info_t *dip, sysino_t sysino, cpuid_t *cpuid) 259 { 260 uint64_t ret; 261 262 DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: dip 0x%p sysino 0x%llx\n", 263 dip, sysino); 264 265 if ((ret = hvio_intr_gettarget(sysino, cpuid)) != H_EOK) { 266 DBG(DBG_LIB_INT, dip, 267 "hvio_intr_gettarget failed, ret 0x%lx\n", ret); 268 return (DDI_FAILURE); 269 } 270 271 DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: cpuid 0x%x\n", cpuid); 272 273 return (DDI_SUCCESS); 274 } 275 276 /*ARGSUSED*/ 277 int 278 px_lib_intr_settarget(dev_info_t *dip, sysino_t sysino, cpuid_t cpuid) 279 { 280 uint64_t ret; 281 282 DBG(DBG_LIB_INT, dip, "px_lib_intr_settarget: dip 0x%p sysino 0x%llx " 283 "cpuid 0x%x\n", dip, sysino, cpuid); 284 285 ret = hvio_intr_settarget(sysino, cpuid); 286 if (ret == H_ECPUERROR) { 287 cmn_err(CE_PANIC, 288 "px_lib_intr_settarget: hvio_intr_settarget failed, " 289 "ret = 0x%lx, cpuid = 0x%x, sysino = 0x%lx\n", ret, 290 cpuid, sysino); 291 } else if (ret != H_EOK) { 292 DBG(DBG_LIB_INT, dip, 293 "hvio_intr_settarget failed, ret 0x%lx\n", ret); 294 return (DDI_FAILURE); 295 } 296 297 return (DDI_SUCCESS); 298 } 299 300 /*ARGSUSED*/ 301 int 302 px_lib_intr_reset(dev_info_t *dip) 303 { 304 px_t *px_p = DIP_TO_STATE(dip); 305 px_ib_t *ib_p = px_p->px_ib_p; 306 px_ino_t *ino_p; 307 308 DBG(DBG_LIB_INT, dip, "px_lib_intr_reset: dip 0x%p\n", dip); 309 310 mutex_enter(&ib_p->ib_ino_lst_mutex); 311 312 /* Reset all Interrupts */ 313 for (ino_p = ib_p->ib_ino_lst; ino_p; ino_p = ino_p->ino_next_p) { 314 if (px_lib_intr_setstate(dip, ino_p->ino_sysino, 315 INTR_IDLE_STATE) != DDI_SUCCESS) 316 return (BF_FATAL); 317 } 318 319 mutex_exit(&ib_p->ib_ino_lst_mutex); 320 321 return (BF_NONE); 322 } 323 324 /*ARGSUSED*/ 325 int 326 px_lib_iommu_map(dev_info_t *dip, tsbid_t tsbid, pages_t pages, 327 io_attributes_t attr, void *addr, size_t pfn_index, int flags) 328 { 329 tsbnum_t tsb_num = PCI_TSBID_TO_TSBNUM(tsbid); 330 tsbindex_t tsb_index = PCI_TSBID_TO_TSBINDEX(tsbid); 331 io_page_list_t *pfns, *pfn_p; 332 pages_t ttes_mapped = 0; 333 int i, err = DDI_SUCCESS; 334 335 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: dip 0x%p tsbid 0x%llx " 336 "pages 0x%x attr 0x%llx addr 0x%p pfn_index 0x%llx flags 0x%x\n", 337 dip, tsbid, pages, attr, addr, pfn_index, flags); 338 339 if ((pfns = pfn_p = kmem_zalloc((pages * sizeof (io_page_list_t)), 340 KM_NOSLEEP)) == NULL) { 341 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: kmem_zalloc failed\n"); 342 return (DDI_FAILURE); 343 } 344 345 for (i = 0; i < pages; i++) 346 pfns[i] = MMU_PTOB(PX_ADDR2PFN(addr, pfn_index, flags, i)); 347 348 /* 349 * If HV VPCI version is 1.1 and higher, pass BDF, phantom function, 350 * and relaxed ordering attributes. Otherwise, pass only read or write 351 * attribute. 352 */ 353 if (px_vpci_min_ver == PX_VPCI_MINOR_VER_0) 354 attr = attr & (PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE); 355 356 while ((ttes_mapped = pfn_p - pfns) < pages) { 357 uintptr_t ra = va_to_pa(pfn_p); 358 pages_t ttes2map; 359 uint64_t ret; 360 361 ttes2map = (MMU_PAGE_SIZE - P2PHASE(ra, MMU_PAGE_SIZE)) >> 3; 362 ra = MMU_PTOB(MMU_BTOP(ra)); 363 364 for (ttes2map = MIN(ttes2map, pages - ttes_mapped); ttes2map; 365 ttes2map -= ttes_mapped, pfn_p += ttes_mapped) { 366 367 ttes_mapped = 0; 368 if ((ret = hvio_iommu_map(DIP_TO_HANDLE(dip), 369 PCI_TSBID(tsb_num, tsb_index + (pfn_p - pfns)), 370 ttes2map, attr, (io_page_list_t *)(ra | 371 ((uintptr_t)pfn_p & MMU_PAGE_OFFSET)), 372 &ttes_mapped)) != H_EOK) { 373 DBG(DBG_LIB_DMA, dip, "hvio_iommu_map failed " 374 "ret 0x%lx\n", ret); 375 376 ttes_mapped = pfn_p - pfns; 377 err = DDI_FAILURE; 378 goto cleanup; 379 } 380 381 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: tsb_num 0x%x " 382 "tsb_index 0x%lx ttes_to_map 0x%lx attr 0x%llx " 383 "ra 0x%p ttes_mapped 0x%x\n", tsb_num, 384 tsb_index + (pfn_p - pfns), ttes2map, attr, 385 ra | ((uintptr_t)pfn_p & MMU_PAGE_OFFSET), 386 ttes_mapped); 387 } 388 } 389 390 cleanup: 391 if ((err == DDI_FAILURE) && ttes_mapped) 392 (void) px_lib_iommu_demap(dip, tsbid, ttes_mapped); 393 394 kmem_free(pfns, pages * sizeof (io_page_list_t)); 395 return (err); 396 } 397 398 /*ARGSUSED*/ 399 int 400 px_lib_iommu_demap(dev_info_t *dip, tsbid_t tsbid, pages_t pages) 401 { 402 tsbnum_t tsb_num = PCI_TSBID_TO_TSBNUM(tsbid); 403 tsbindex_t tsb_index = PCI_TSBID_TO_TSBINDEX(tsbid); 404 pages_t ttes2demap, ttes_demapped = 0; 405 uint64_t ret; 406 407 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: dip 0x%p tsbid 0x%llx " 408 "pages 0x%x\n", dip, tsbid, pages); 409 410 for (ttes2demap = pages; ttes2demap; 411 ttes2demap -= ttes_demapped, tsb_index += ttes_demapped) { 412 if ((ret = hvio_iommu_demap(DIP_TO_HANDLE(dip), 413 PCI_TSBID(tsb_num, tsb_index), ttes2demap, 414 &ttes_demapped)) != H_EOK) { 415 DBG(DBG_LIB_DMA, dip, "hvio_iommu_demap failed, " 416 "ret 0x%lx\n", ret); 417 418 return (DDI_FAILURE); 419 } 420 421 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: tsb_num 0x%x " 422 "tsb_index 0x%lx ttes_to_demap 0x%lx ttes_demapped 0x%x\n", 423 tsb_num, tsb_index, ttes2demap, ttes_demapped); 424 } 425 426 return (DDI_SUCCESS); 427 } 428 429 /*ARGSUSED*/ 430 int 431 px_lib_iommu_getmap(dev_info_t *dip, tsbid_t tsbid, io_attributes_t *attr_p, 432 r_addr_t *r_addr_p) 433 { 434 uint64_t ret; 435 436 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: dip 0x%p tsbid 0x%llx\n", 437 dip, tsbid); 438 439 if ((ret = hvio_iommu_getmap(DIP_TO_HANDLE(dip), tsbid, 440 attr_p, r_addr_p)) != H_EOK) { 441 DBG(DBG_LIB_DMA, dip, 442 "hvio_iommu_getmap failed, ret 0x%lx\n", ret); 443 444 return ((ret == H_ENOMAP) ? DDI_DMA_NOMAPPING:DDI_FAILURE); 445 } 446 447 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: attr 0x%llx " 448 "r_addr 0x%llx\n", *attr_p, *r_addr_p); 449 450 return (DDI_SUCCESS); 451 } 452 453 /*ARGSUSED*/ 454 uint64_t 455 px_get_rng_parent_hi_mask(px_t *px_p) 456 { 457 return (PX_RANGE_PROP_MASK); 458 } 459 460 /* 461 * Checks dma attributes against system bypass ranges 462 * A sun4v device must be capable of generating the entire 64-bit 463 * address in order to perform bypass DMA. 464 */ 465 /*ARGSUSED*/ 466 int 467 px_lib_dma_bypass_rngchk(dev_info_t *dip, ddi_dma_attr_t *attr_p, 468 uint64_t *lo_p, uint64_t *hi_p) 469 { 470 if ((attr_p->dma_attr_addr_lo != 0ull) || 471 (attr_p->dma_attr_addr_hi != UINT64_MAX)) { 472 473 return (DDI_DMA_BADATTR); 474 } 475 476 *lo_p = 0ull; 477 *hi_p = UINT64_MAX; 478 479 return (DDI_SUCCESS); 480 } 481 482 483 /*ARGSUSED*/ 484 int 485 px_lib_iommu_getbypass(dev_info_t *dip, r_addr_t ra, io_attributes_t attr, 486 io_addr_t *io_addr_p) 487 { 488 uint64_t ret; 489 490 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: dip 0x%p ra 0x%llx " 491 "attr 0x%llx\n", dip, ra, attr); 492 493 if ((ret = hvio_iommu_getbypass(DIP_TO_HANDLE(dip), ra, 494 attr, io_addr_p)) != H_EOK) { 495 DBG(DBG_LIB_DMA, dip, 496 "hvio_iommu_getbypass failed, ret 0x%lx\n", ret); 497 return (ret == H_ENOTSUPPORTED ? DDI_ENOTSUP : DDI_FAILURE); 498 } 499 500 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: io_addr 0x%llx\n", 501 *io_addr_p); 502 503 return (DDI_SUCCESS); 504 } 505 506 /* 507 * Returns any needed IO address bit(s) for relaxed ordering in IOMMU 508 * bypass mode. 509 */ 510 /* ARGSUSED */ 511 uint64_t 512 px_lib_ro_bypass(dev_info_t *dip, io_attributes_t attr, uint64_t ioaddr) 513 { 514 return (ioaddr); 515 } 516 517 /*ARGSUSED*/ 518 int 519 px_lib_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 520 off_t off, size_t len, uint_t cache_flags) 521 { 522 ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; 523 uint64_t sync_dir; 524 size_t bytes_synced; 525 int end, idx; 526 off_t pg_off; 527 devhandle_t hdl = DIP_TO_HANDLE(dip); /* need to cache hdl */ 528 529 DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: dip 0x%p rdip 0x%p " 530 "handle 0x%llx off 0x%x len 0x%x flags 0x%x\n", 531 dip, rdip, handle, off, len, cache_flags); 532 533 if (!(mp->dmai_flags & PX_DMAI_FLAGS_INUSE)) { 534 cmn_err(CE_WARN, "%s%d: Unbound dma handle %p.", 535 ddi_driver_name(rdip), ddi_get_instance(rdip), (void *)mp); 536 return (DDI_FAILURE); 537 } 538 539 if (mp->dmai_flags & PX_DMAI_FLAGS_NOSYNC) 540 return (DDI_SUCCESS); 541 542 if (!len) 543 len = mp->dmai_size; 544 545 if (mp->dmai_rflags & DDI_DMA_READ) 546 sync_dir = HVIO_DMA_SYNC_DIR_FROM_DEV; 547 else 548 sync_dir = HVIO_DMA_SYNC_DIR_TO_DEV; 549 550 if (force_sync_icache_after_dma == 0 && !icache_is_coherent) 551 sync_dir |= HVIO_DMA_SYNC_DIR_NO_ICACHE_FLUSH; 552 553 off += mp->dmai_offset; 554 pg_off = off & MMU_PAGEOFFSET; 555 556 DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: page offset %x size %x\n", 557 pg_off, len); 558 559 /* sync on page basis */ 560 end = MMU_BTOPR(off + len - 1); 561 for (idx = MMU_BTOP(off); idx < end; idx++, 562 len -= bytes_synced, pg_off = 0) { 563 size_t bytes_to_sync = MIN(len, MMU_PAGESIZE - pg_off); 564 565 while (hvio_dma_sync(hdl, 566 MMU_PTOB(PX_GET_MP_PFN(mp, idx)) + pg_off, 567 bytes_to_sync, sync_dir, &bytes_synced) != H_EOK) { 568 569 if (!(sync_dir & HVIO_DMA_SYNC_DIR_NO_ICACHE_FLUSH)) { 570 bytes_synced = 0; 571 break; 572 } 573 574 /* 575 * Some versions of firmware do not support 576 * this sync_dir flag. If the call fails clear 577 * the flag and retry the call. Also, set the 578 * global so that we dont set the sync_dir 579 * flag again. 580 */ 581 sync_dir &= ~HVIO_DMA_SYNC_DIR_NO_ICACHE_FLUSH; 582 force_sync_icache_after_dma = 1; 583 } 584 585 DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: Called hvio_dma_sync " 586 "ra = %p bytes to sync = %x bytes synced %x\n", 587 MMU_PTOB(PX_GET_MP_PFN(mp, idx)) + pg_off, bytes_to_sync, 588 bytes_synced); 589 590 if (bytes_to_sync != bytes_synced) 591 break; 592 } 593 594 return (len ? DDI_FAILURE : DDI_SUCCESS); 595 } 596 597 598 /* 599 * MSIQ Functions: 600 */ 601 602 /*ARGSUSED*/ 603 int 604 px_lib_msiq_init(dev_info_t *dip) 605 { 606 px_t *px_p = DIP_TO_STATE(dip); 607 px_msiq_state_t *msiq_state_p = &px_p->px_ib_p->ib_msiq_state; 608 r_addr_t ra; 609 size_t msiq_size; 610 uint_t rec_cnt; 611 int i, err = DDI_SUCCESS; 612 uint64_t ret; 613 614 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_init: dip 0x%p\n", dip); 615 616 msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t); 617 618 /* sun4v requires all EQ allocation to be on q size boundary */ 619 if ((msiq_state_p->msiq_buf_p = contig_mem_alloc_align( 620 msiq_state_p->msiq_cnt * msiq_size, msiq_size)) == NULL) { 621 DBG(DBG_LIB_MSIQ, dip, 622 "px_lib_msiq_init: Contig alloc failed\n"); 623 624 return (DDI_FAILURE); 625 } 626 627 for (i = 0; i < msiq_state_p->msiq_cnt; i++) { 628 msiq_state_p->msiq_p[i].msiq_base_p = (msiqhead_t *) 629 ((caddr_t)msiq_state_p->msiq_buf_p + (i * msiq_size)); 630 631 ra = (r_addr_t)va_to_pa((caddr_t)msiq_state_p->msiq_buf_p + 632 (i * msiq_size)); 633 634 if ((ret = hvio_msiq_conf(DIP_TO_HANDLE(dip), 635 (i + msiq_state_p->msiq_1st_msiq_id), 636 ra, msiq_state_p->msiq_rec_cnt)) != H_EOK) { 637 DBG(DBG_LIB_MSIQ, dip, 638 "hvio_msiq_conf failed, ret 0x%lx\n", ret); 639 err = DDI_FAILURE; 640 break; 641 } 642 643 if ((err = px_lib_msiq_info(dip, 644 (i + msiq_state_p->msiq_1st_msiq_id), 645 &ra, &rec_cnt)) != DDI_SUCCESS) { 646 DBG(DBG_LIB_MSIQ, dip, 647 "px_lib_msiq_info failed, ret 0x%x\n", err); 648 err = DDI_FAILURE; 649 break; 650 } 651 652 DBG(DBG_LIB_MSIQ, dip, 653 "px_lib_msiq_init: ra 0x%p rec_cnt 0x%x\n", ra, rec_cnt); 654 } 655 656 return (err); 657 } 658 659 /*ARGSUSED*/ 660 int 661 px_lib_msiq_fini(dev_info_t *dip) 662 { 663 px_t *px_p = DIP_TO_STATE(dip); 664 px_msiq_state_t *msiq_state_p = &px_p->px_ib_p->ib_msiq_state; 665 size_t msiq_size; 666 667 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_fini: dip 0x%p\n", dip); 668 msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t); 669 670 if (msiq_state_p->msiq_buf_p != NULL) 671 contig_mem_free(msiq_state_p->msiq_buf_p, 672 msiq_state_p->msiq_cnt * msiq_size); 673 674 return (DDI_SUCCESS); 675 } 676 677 /*ARGSUSED*/ 678 int 679 px_lib_msiq_info(dev_info_t *dip, msiqid_t msiq_id, r_addr_t *ra_p, 680 uint_t *msiq_rec_cnt_p) 681 { 682 uint64_t ret; 683 684 DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: dip 0x%p msiq_id 0x%x\n", 685 dip, msiq_id); 686 687 if ((ret = hvio_msiq_info(DIP_TO_HANDLE(dip), 688 msiq_id, ra_p, msiq_rec_cnt_p)) != H_EOK) { 689 DBG(DBG_LIB_MSIQ, dip, 690 "hvio_msiq_info failed, ret 0x%lx\n", ret); 691 return (DDI_FAILURE); 692 } 693 694 DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: ra_p 0x%p msiq_rec_cnt 0x%x\n", 695 ra_p, *msiq_rec_cnt_p); 696 697 return (DDI_SUCCESS); 698 } 699 700 /*ARGSUSED*/ 701 int 702 px_lib_msiq_getvalid(dev_info_t *dip, msiqid_t msiq_id, 703 pci_msiq_valid_state_t *msiq_valid_state) 704 { 705 uint64_t ret; 706 707 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: dip 0x%p msiq_id 0x%x\n", 708 dip, msiq_id); 709 710 if ((ret = hvio_msiq_getvalid(DIP_TO_HANDLE(dip), 711 msiq_id, msiq_valid_state)) != H_EOK) { 712 DBG(DBG_LIB_MSIQ, dip, 713 "hvio_msiq_getvalid failed, ret 0x%lx\n", ret); 714 return (DDI_FAILURE); 715 } 716 717 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: msiq_valid_state 0x%x\n", 718 *msiq_valid_state); 719 720 return (DDI_SUCCESS); 721 } 722 723 /*ARGSUSED*/ 724 int 725 px_lib_msiq_setvalid(dev_info_t *dip, msiqid_t msiq_id, 726 pci_msiq_valid_state_t msiq_valid_state) 727 { 728 uint64_t ret; 729 730 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setvalid: dip 0x%p msiq_id 0x%x " 731 "msiq_valid_state 0x%x\n", dip, msiq_id, msiq_valid_state); 732 733 if ((ret = hvio_msiq_setvalid(DIP_TO_HANDLE(dip), 734 msiq_id, msiq_valid_state)) != H_EOK) { 735 DBG(DBG_LIB_MSIQ, dip, 736 "hvio_msiq_setvalid failed, ret 0x%lx\n", ret); 737 return (DDI_FAILURE); 738 } 739 740 return (DDI_SUCCESS); 741 } 742 743 /*ARGSUSED*/ 744 int 745 px_lib_msiq_getstate(dev_info_t *dip, msiqid_t msiq_id, 746 pci_msiq_state_t *msiq_state) 747 { 748 uint64_t ret; 749 750 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: dip 0x%p msiq_id 0x%x\n", 751 dip, msiq_id); 752 753 if ((ret = hvio_msiq_getstate(DIP_TO_HANDLE(dip), 754 msiq_id, msiq_state)) != H_EOK) { 755 DBG(DBG_LIB_MSIQ, dip, 756 "hvio_msiq_getstate failed, ret 0x%lx\n", ret); 757 return (DDI_FAILURE); 758 } 759 760 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: msiq_state 0x%x\n", 761 *msiq_state); 762 763 return (DDI_SUCCESS); 764 } 765 766 /*ARGSUSED*/ 767 int 768 px_lib_msiq_setstate(dev_info_t *dip, msiqid_t msiq_id, 769 pci_msiq_state_t msiq_state) 770 { 771 uint64_t ret; 772 773 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setstate: dip 0x%p msiq_id 0x%x " 774 "msiq_state 0x%x\n", dip, msiq_id, msiq_state); 775 776 if ((ret = hvio_msiq_setstate(DIP_TO_HANDLE(dip), 777 msiq_id, msiq_state)) != H_EOK) { 778 DBG(DBG_LIB_MSIQ, dip, 779 "hvio_msiq_setstate failed, ret 0x%lx\n", ret); 780 return (DDI_FAILURE); 781 } 782 783 return (DDI_SUCCESS); 784 } 785 786 /*ARGSUSED*/ 787 int 788 px_lib_msiq_gethead(dev_info_t *dip, msiqid_t msiq_id, 789 msiqhead_t *msiq_head_p) 790 { 791 uint64_t ret; 792 793 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: dip 0x%p msiq_id 0x%x\n", 794 dip, msiq_id); 795 796 if ((ret = hvio_msiq_gethead(DIP_TO_HANDLE(dip), 797 msiq_id, msiq_head_p)) != H_EOK) { 798 DBG(DBG_LIB_MSIQ, dip, 799 "hvio_msiq_gethead failed, ret 0x%lx\n", ret); 800 return (DDI_FAILURE); 801 } 802 803 *msiq_head_p = (*msiq_head_p / sizeof (msiq_rec_t)); 804 805 DBG(DBG_LIB_MSIQ, dip, "px_msiq_gethead: msiq_head 0x%x\n", 806 *msiq_head_p); 807 808 return (DDI_SUCCESS); 809 } 810 811 /*ARGSUSED*/ 812 int 813 px_lib_msiq_sethead(dev_info_t *dip, msiqid_t msiq_id, 814 msiqhead_t msiq_head) 815 { 816 uint64_t ret; 817 818 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_sethead: dip 0x%p msiq_id 0x%x " 819 "msiq_head 0x%x\n", dip, msiq_id, msiq_head); 820 821 if ((ret = hvio_msiq_sethead(DIP_TO_HANDLE(dip), 822 msiq_id, msiq_head * sizeof (msiq_rec_t))) != H_EOK) { 823 DBG(DBG_LIB_MSIQ, dip, 824 "hvio_msiq_sethead failed, ret 0x%lx\n", ret); 825 return (DDI_FAILURE); 826 } 827 828 return (DDI_SUCCESS); 829 } 830 831 /*ARGSUSED*/ 832 int 833 px_lib_msiq_gettail(dev_info_t *dip, msiqid_t msiq_id, 834 msiqtail_t *msiq_tail_p) 835 { 836 uint64_t ret; 837 838 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: dip 0x%p msiq_id 0x%x\n", 839 dip, msiq_id); 840 841 if ((ret = hvio_msiq_gettail(DIP_TO_HANDLE(dip), 842 msiq_id, msiq_tail_p)) != H_EOK) { 843 DBG(DBG_LIB_MSIQ, dip, 844 "hvio_msiq_gettail failed, ret 0x%lx\n", ret); 845 return (DDI_FAILURE); 846 } 847 848 *msiq_tail_p = (*msiq_tail_p / sizeof (msiq_rec_t)); 849 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: msiq_tail 0x%x\n", 850 *msiq_tail_p); 851 852 return (DDI_SUCCESS); 853 } 854 855 /*ARGSUSED*/ 856 void 857 px_lib_get_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p, 858 msiq_rec_t *msiq_rec_p) 859 { 860 msiq_rec_t *curr_msiq_rec_p = (msiq_rec_t *)msiq_head_p; 861 862 DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: dip 0x%p\n", dip); 863 864 if (!curr_msiq_rec_p->msiq_rec_type) { 865 /* Set msiq_rec_type to zero */ 866 msiq_rec_p->msiq_rec_type = 0; 867 868 return; 869 } 870 871 *msiq_rec_p = *curr_msiq_rec_p; 872 } 873 874 /*ARGSUSED*/ 875 void 876 px_lib_clr_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p) 877 { 878 msiq_rec_t *curr_msiq_rec_p = (msiq_rec_t *)msiq_head_p; 879 880 DBG(DBG_LIB_MSIQ, dip, "px_lib_clr_msiq_rec: dip 0x%p\n", dip); 881 882 /* Zero out msiq_rec_type field */ 883 curr_msiq_rec_p->msiq_rec_type = 0; 884 } 885 886 /* 887 * MSI Functions: 888 */ 889 890 /*ARGSUSED*/ 891 int 892 px_lib_msi_init(dev_info_t *dip) 893 { 894 DBG(DBG_LIB_MSI, dip, "px_lib_msi_init: dip 0x%p\n", dip); 895 896 /* Noop */ 897 return (DDI_SUCCESS); 898 } 899 900 /*ARGSUSED*/ 901 int 902 px_lib_msi_getmsiq(dev_info_t *dip, msinum_t msi_num, 903 msiqid_t *msiq_id) 904 { 905 uint64_t ret; 906 907 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: dip 0x%p msi_num 0x%x\n", 908 dip, msi_num); 909 910 if ((ret = hvio_msi_getmsiq(DIP_TO_HANDLE(dip), 911 msi_num, msiq_id)) != H_EOK) { 912 DBG(DBG_LIB_MSI, dip, 913 "hvio_msi_getmsiq failed, ret 0x%lx\n", ret); 914 return (DDI_FAILURE); 915 } 916 917 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: msiq_id 0x%x\n", 918 *msiq_id); 919 920 return (DDI_SUCCESS); 921 } 922 923 /*ARGSUSED*/ 924 int 925 px_lib_msi_setmsiq(dev_info_t *dip, msinum_t msi_num, 926 msiqid_t msiq_id, msi_type_t msitype) 927 { 928 uint64_t ret; 929 930 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setmsiq: dip 0x%p msi_num 0x%x " 931 "msq_id 0x%x\n", dip, msi_num, msiq_id); 932 933 if ((ret = hvio_msi_setmsiq(DIP_TO_HANDLE(dip), 934 msi_num, msiq_id, msitype)) != H_EOK) { 935 DBG(DBG_LIB_MSI, dip, 936 "hvio_msi_setmsiq failed, ret 0x%lx\n", ret); 937 return (DDI_FAILURE); 938 } 939 940 return (DDI_SUCCESS); 941 } 942 943 /*ARGSUSED*/ 944 int 945 px_lib_msi_getvalid(dev_info_t *dip, msinum_t msi_num, 946 pci_msi_valid_state_t *msi_valid_state) 947 { 948 uint64_t ret; 949 950 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: dip 0x%p msi_num 0x%x\n", 951 dip, msi_num); 952 953 if ((ret = hvio_msi_getvalid(DIP_TO_HANDLE(dip), 954 msi_num, msi_valid_state)) != H_EOK) { 955 DBG(DBG_LIB_MSI, dip, 956 "hvio_msi_getvalid failed, ret 0x%lx\n", ret); 957 return (DDI_FAILURE); 958 } 959 960 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: msiq_id 0x%x\n", 961 *msi_valid_state); 962 963 return (DDI_SUCCESS); 964 } 965 966 /*ARGSUSED*/ 967 int 968 px_lib_msi_setvalid(dev_info_t *dip, msinum_t msi_num, 969 pci_msi_valid_state_t msi_valid_state) 970 { 971 uint64_t ret; 972 973 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setvalid: dip 0x%p msi_num 0x%x " 974 "msi_valid_state 0x%x\n", dip, msi_num, msi_valid_state); 975 976 if ((ret = hvio_msi_setvalid(DIP_TO_HANDLE(dip), 977 msi_num, msi_valid_state)) != H_EOK) { 978 DBG(DBG_LIB_MSI, dip, 979 "hvio_msi_setvalid failed, ret 0x%lx\n", ret); 980 return (DDI_FAILURE); 981 } 982 983 return (DDI_SUCCESS); 984 } 985 986 /*ARGSUSED*/ 987 int 988 px_lib_msi_getstate(dev_info_t *dip, msinum_t msi_num, 989 pci_msi_state_t *msi_state) 990 { 991 uint64_t ret; 992 993 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: dip 0x%p msi_num 0x%x\n", 994 dip, msi_num); 995 996 if ((ret = hvio_msi_getstate(DIP_TO_HANDLE(dip), 997 msi_num, msi_state)) != H_EOK) { 998 DBG(DBG_LIB_MSI, dip, 999 "hvio_msi_getstate failed, ret 0x%lx\n", ret); 1000 return (DDI_FAILURE); 1001 } 1002 1003 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: msi_state 0x%x\n", 1004 *msi_state); 1005 1006 return (DDI_SUCCESS); 1007 } 1008 1009 /*ARGSUSED*/ 1010 int 1011 px_lib_msi_setstate(dev_info_t *dip, msinum_t msi_num, 1012 pci_msi_state_t msi_state) 1013 { 1014 uint64_t ret; 1015 1016 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setstate: dip 0x%p msi_num 0x%x " 1017 "msi_state 0x%x\n", dip, msi_num, msi_state); 1018 1019 if ((ret = hvio_msi_setstate(DIP_TO_HANDLE(dip), 1020 msi_num, msi_state)) != H_EOK) { 1021 DBG(DBG_LIB_MSI, dip, 1022 "hvio_msi_setstate failed, ret 0x%lx\n", ret); 1023 return (DDI_FAILURE); 1024 } 1025 1026 return (DDI_SUCCESS); 1027 } 1028 1029 /* 1030 * MSG Functions: 1031 */ 1032 1033 /*ARGSUSED*/ 1034 int 1035 px_lib_msg_getmsiq(dev_info_t *dip, pcie_msg_type_t msg_type, 1036 msiqid_t *msiq_id) 1037 { 1038 uint64_t ret; 1039 1040 DBG(DBG_LIB_MSG, dip, "px_lib_msg_getmsiq: dip 0x%p msg_type 0x%x\n", 1041 dip, msg_type); 1042 1043 if ((ret = hvio_msg_getmsiq(DIP_TO_HANDLE(dip), 1044 msg_type, msiq_id)) != H_EOK) { 1045 DBG(DBG_LIB_MSG, dip, 1046 "hvio_msg_getmsiq failed, ret 0x%lx\n", ret); 1047 return (DDI_FAILURE); 1048 } 1049 1050 DBG(DBG_LIB_MSI, dip, "px_lib_msg_getmsiq: msiq_id 0x%x\n", 1051 *msiq_id); 1052 1053 return (DDI_SUCCESS); 1054 } 1055 1056 /*ARGSUSED*/ 1057 int 1058 px_lib_msg_setmsiq(dev_info_t *dip, pcie_msg_type_t msg_type, 1059 msiqid_t msiq_id) 1060 { 1061 uint64_t ret; 1062 1063 DBG(DBG_LIB_MSG, dip, "px_lib_msg_setmsiq: dip 0x%p msg_type 0x%x " 1064 "msq_id 0x%x\n", dip, msg_type, msiq_id); 1065 1066 if ((ret = hvio_msg_setmsiq(DIP_TO_HANDLE(dip), 1067 msg_type, msiq_id)) != H_EOK) { 1068 DBG(DBG_LIB_MSG, dip, 1069 "hvio_msg_setmsiq failed, ret 0x%lx\n", ret); 1070 return (DDI_FAILURE); 1071 } 1072 1073 return (DDI_SUCCESS); 1074 } 1075 1076 /*ARGSUSED*/ 1077 int 1078 px_lib_msg_getvalid(dev_info_t *dip, pcie_msg_type_t msg_type, 1079 pcie_msg_valid_state_t *msg_valid_state) 1080 { 1081 uint64_t ret; 1082 1083 DBG(DBG_LIB_MSG, dip, "px_lib_msg_getvalid: dip 0x%p msg_type 0x%x\n", 1084 dip, msg_type); 1085 1086 if ((ret = hvio_msg_getvalid(DIP_TO_HANDLE(dip), msg_type, 1087 msg_valid_state)) != H_EOK) { 1088 DBG(DBG_LIB_MSG, dip, 1089 "hvio_msg_getvalid failed, ret 0x%lx\n", ret); 1090 return (DDI_FAILURE); 1091 } 1092 1093 DBG(DBG_LIB_MSI, dip, "px_lib_msg_getvalid: msg_valid_state 0x%x\n", 1094 *msg_valid_state); 1095 1096 return (DDI_SUCCESS); 1097 } 1098 1099 /*ARGSUSED*/ 1100 int 1101 px_lib_msg_setvalid(dev_info_t *dip, pcie_msg_type_t msg_type, 1102 pcie_msg_valid_state_t msg_valid_state) 1103 { 1104 uint64_t ret; 1105 1106 DBG(DBG_LIB_MSG, dip, "px_lib_msg_setvalid: dip 0x%p msg_type 0x%x " 1107 "msg_valid_state 0x%x\n", dip, msg_type, msg_valid_state); 1108 1109 if ((ret = hvio_msg_setvalid(DIP_TO_HANDLE(dip), msg_type, 1110 msg_valid_state)) != H_EOK) { 1111 DBG(DBG_LIB_MSG, dip, 1112 "hvio_msg_setvalid failed, ret 0x%lx\n", ret); 1113 return (DDI_FAILURE); 1114 } 1115 1116 return (DDI_SUCCESS); 1117 } 1118 1119 /* 1120 * Suspend/Resume Functions: 1121 * Currently unsupported by hypervisor and all functions are noops. 1122 */ 1123 /*ARGSUSED*/ 1124 int 1125 px_lib_suspend(dev_info_t *dip) 1126 { 1127 DBG(DBG_ATTACH, dip, "px_lib_suspend: Not supported\n"); 1128 1129 /* Not supported */ 1130 return (DDI_FAILURE); 1131 } 1132 1133 /*ARGSUSED*/ 1134 void 1135 px_lib_resume(dev_info_t *dip) 1136 { 1137 DBG(DBG_ATTACH, dip, "px_lib_resume: Not supported\n"); 1138 1139 /* Noop */ 1140 } 1141 1142 /* 1143 * Misc Functions: 1144 * Currently unsupported by hypervisor and all functions are noops. 1145 */ 1146 /*ARGSUSED*/ 1147 static int 1148 px_lib_config_get(dev_info_t *dip, pci_device_t bdf, pci_config_offset_t off, 1149 uint8_t size, pci_cfg_data_t *data_p) 1150 { 1151 uint64_t ret; 1152 1153 DBG(DBG_LIB_CFG, dip, "px_lib_config_get: dip 0x%p, bdf 0x%llx " 1154 "off 0x%x size 0x%x\n", dip, bdf, off, size); 1155 1156 if ((ret = hvio_config_get(DIP_TO_HANDLE(dip), bdf, off, 1157 size, data_p)) != H_EOK) { 1158 DBG(DBG_LIB_CFG, dip, 1159 "hvio_config_get failed, ret 0x%lx\n", ret); 1160 return (DDI_FAILURE); 1161 } 1162 DBG(DBG_LIB_CFG, dip, "px_config_get: data 0x%x\n", data_p->dw); 1163 1164 return (DDI_SUCCESS); 1165 } 1166 1167 /*ARGSUSED*/ 1168 static int 1169 px_lib_config_put(dev_info_t *dip, pci_device_t bdf, pci_config_offset_t off, 1170 uint8_t size, pci_cfg_data_t data) 1171 { 1172 uint64_t ret; 1173 1174 DBG(DBG_LIB_CFG, dip, "px_lib_config_put: dip 0x%p, bdf 0x%llx " 1175 "off 0x%x size 0x%x data 0x%llx\n", dip, bdf, off, size, data.qw); 1176 1177 if ((ret = hvio_config_put(DIP_TO_HANDLE(dip), bdf, off, 1178 size, data)) != H_EOK) { 1179 DBG(DBG_LIB_CFG, dip, 1180 "hvio_config_put failed, ret 0x%lx\n", ret); 1181 return (DDI_FAILURE); 1182 } 1183 1184 return (DDI_SUCCESS); 1185 } 1186 1187 static uint32_t 1188 px_pci_config_get(ddi_acc_impl_t *handle, uint32_t *addr, int size) 1189 { 1190 px_config_acc_pvt_t *px_pvt = (px_config_acc_pvt_t *) 1191 handle->ahi_common.ah_bus_private; 1192 uint32_t pci_dev_addr = px_pvt->raddr; 1193 uint32_t vaddr = px_pvt->vaddr; 1194 uint16_t off = (uint16_t)(uintptr_t)(addr - vaddr) & 0xfff; 1195 uint32_t rdata = 0; 1196 1197 if (px_lib_config_get(px_pvt->dip, pci_dev_addr, off, 1198 size, (pci_cfg_data_t *)&rdata) != DDI_SUCCESS) 1199 /* XXX update error kstats */ 1200 return (0xffffffff); 1201 return (rdata); 1202 } 1203 1204 static void 1205 px_pci_config_put(ddi_acc_impl_t *handle, uint32_t *addr, 1206 int size, pci_cfg_data_t wdata) 1207 { 1208 px_config_acc_pvt_t *px_pvt = (px_config_acc_pvt_t *) 1209 handle->ahi_common.ah_bus_private; 1210 uint32_t pci_dev_addr = px_pvt->raddr; 1211 uint32_t vaddr = px_pvt->vaddr; 1212 uint16_t off = (uint16_t)(uintptr_t)(addr - vaddr) & 0xfff; 1213 1214 if (px_lib_config_put(px_pvt->dip, pci_dev_addr, off, 1215 size, wdata) != DDI_SUCCESS) { 1216 /*EMPTY*/ 1217 /* XXX update error kstats */ 1218 } 1219 } 1220 1221 static uint8_t 1222 px_pci_config_get8(ddi_acc_impl_t *handle, uint8_t *addr) 1223 { 1224 return ((uint8_t)px_pci_config_get(handle, (uint32_t *)addr, 1)); 1225 } 1226 1227 static uint16_t 1228 px_pci_config_get16(ddi_acc_impl_t *handle, uint16_t *addr) 1229 { 1230 return ((uint16_t)px_pci_config_get(handle, (uint32_t *)addr, 2)); 1231 } 1232 1233 static uint32_t 1234 px_pci_config_get32(ddi_acc_impl_t *handle, uint32_t *addr) 1235 { 1236 return ((uint32_t)px_pci_config_get(handle, (uint32_t *)addr, 4)); 1237 } 1238 1239 static uint64_t 1240 px_pci_config_get64(ddi_acc_impl_t *handle, uint64_t *addr) 1241 { 1242 uint32_t rdatah, rdatal; 1243 1244 rdatal = (uint32_t)px_pci_config_get(handle, (uint32_t *)addr, 4); 1245 rdatah = (uint32_t)px_pci_config_get(handle, 1246 (uint32_t *)((char *)addr+4), 4); 1247 return (((uint64_t)rdatah << 32) | rdatal); 1248 } 1249 1250 static void 1251 px_pci_config_put8(ddi_acc_impl_t *handle, uint8_t *addr, uint8_t data) 1252 { 1253 pci_cfg_data_t wdata = { 0 }; 1254 1255 wdata.qw = (uint8_t)data; 1256 px_pci_config_put(handle, (uint32_t *)addr, 1, wdata); 1257 } 1258 1259 static void 1260 px_pci_config_put16(ddi_acc_impl_t *handle, uint16_t *addr, uint16_t data) 1261 { 1262 pci_cfg_data_t wdata = { 0 }; 1263 1264 wdata.qw = (uint16_t)data; 1265 px_pci_config_put(handle, (uint32_t *)addr, 2, wdata); 1266 } 1267 1268 static void 1269 px_pci_config_put32(ddi_acc_impl_t *handle, uint32_t *addr, uint32_t data) 1270 { 1271 pci_cfg_data_t wdata = { 0 }; 1272 1273 wdata.qw = (uint32_t)data; 1274 px_pci_config_put(handle, (uint32_t *)addr, 4, wdata); 1275 } 1276 1277 static void 1278 px_pci_config_put64(ddi_acc_impl_t *handle, uint64_t *addr, uint64_t data) 1279 { 1280 pci_cfg_data_t wdata = { 0 }; 1281 1282 wdata.qw = (uint32_t)(data & 0xffffffff); 1283 px_pci_config_put(handle, (uint32_t *)addr, 4, wdata); 1284 wdata.qw = (uint32_t)((data >> 32) & 0xffffffff); 1285 px_pci_config_put(handle, (uint32_t *)((char *)addr+4), 4, wdata); 1286 } 1287 1288 static void 1289 px_pci_config_rep_get8(ddi_acc_impl_t *handle, uint8_t *host_addr, 1290 uint8_t *dev_addr, size_t repcount, uint_t flags) 1291 { 1292 if (flags == DDI_DEV_AUTOINCR) 1293 for (; repcount; repcount--) 1294 *host_addr++ = px_pci_config_get8(handle, dev_addr++); 1295 else 1296 for (; repcount; repcount--) 1297 *host_addr++ = px_pci_config_get8(handle, dev_addr); 1298 } 1299 1300 /* 1301 * Function to rep read 16 bit data off the PCI configuration space behind 1302 * the 21554's host interface. 1303 */ 1304 static void 1305 px_pci_config_rep_get16(ddi_acc_impl_t *handle, uint16_t *host_addr, 1306 uint16_t *dev_addr, size_t repcount, uint_t flags) 1307 { 1308 if (flags == DDI_DEV_AUTOINCR) 1309 for (; repcount; repcount--) 1310 *host_addr++ = px_pci_config_get16(handle, dev_addr++); 1311 else 1312 for (; repcount; repcount--) 1313 *host_addr++ = px_pci_config_get16(handle, dev_addr); 1314 } 1315 1316 /* 1317 * Function to rep read 32 bit data off the PCI configuration space behind 1318 * the 21554's host interface. 1319 */ 1320 static void 1321 px_pci_config_rep_get32(ddi_acc_impl_t *handle, uint32_t *host_addr, 1322 uint32_t *dev_addr, size_t repcount, uint_t flags) 1323 { 1324 if (flags == DDI_DEV_AUTOINCR) 1325 for (; repcount; repcount--) 1326 *host_addr++ = px_pci_config_get32(handle, dev_addr++); 1327 else 1328 for (; repcount; repcount--) 1329 *host_addr++ = px_pci_config_get32(handle, dev_addr); 1330 } 1331 1332 /* 1333 * Function to rep read 64 bit data off the PCI configuration space behind 1334 * the 21554's host interface. 1335 */ 1336 static void 1337 px_pci_config_rep_get64(ddi_acc_impl_t *handle, uint64_t *host_addr, 1338 uint64_t *dev_addr, size_t repcount, uint_t flags) 1339 { 1340 if (flags == DDI_DEV_AUTOINCR) 1341 for (; repcount; repcount--) 1342 *host_addr++ = px_pci_config_get64(handle, dev_addr++); 1343 else 1344 for (; repcount; repcount--) 1345 *host_addr++ = px_pci_config_get64(handle, dev_addr); 1346 } 1347 1348 /* 1349 * Function to rep write 8 bit data into the PCI configuration space behind 1350 * the 21554's host interface. 1351 */ 1352 static void 1353 px_pci_config_rep_put8(ddi_acc_impl_t *handle, uint8_t *host_addr, 1354 uint8_t *dev_addr, size_t repcount, uint_t flags) 1355 { 1356 if (flags == DDI_DEV_AUTOINCR) 1357 for (; repcount; repcount--) 1358 px_pci_config_put8(handle, dev_addr++, *host_addr++); 1359 else 1360 for (; repcount; repcount--) 1361 px_pci_config_put8(handle, dev_addr, *host_addr++); 1362 } 1363 1364 /* 1365 * Function to rep write 16 bit data into the PCI configuration space behind 1366 * the 21554's host interface. 1367 */ 1368 static void 1369 px_pci_config_rep_put16(ddi_acc_impl_t *handle, uint16_t *host_addr, 1370 uint16_t *dev_addr, size_t repcount, uint_t flags) 1371 { 1372 if (flags == DDI_DEV_AUTOINCR) 1373 for (; repcount; repcount--) 1374 px_pci_config_put16(handle, dev_addr++, *host_addr++); 1375 else 1376 for (; repcount; repcount--) 1377 px_pci_config_put16(handle, dev_addr, *host_addr++); 1378 } 1379 1380 /* 1381 * Function to rep write 32 bit data into the PCI configuration space behind 1382 * the 21554's host interface. 1383 */ 1384 static void 1385 px_pci_config_rep_put32(ddi_acc_impl_t *handle, uint32_t *host_addr, 1386 uint32_t *dev_addr, size_t repcount, uint_t flags) 1387 { 1388 if (flags == DDI_DEV_AUTOINCR) 1389 for (; repcount; repcount--) 1390 px_pci_config_put32(handle, dev_addr++, *host_addr++); 1391 else 1392 for (; repcount; repcount--) 1393 px_pci_config_put32(handle, dev_addr, *host_addr++); 1394 } 1395 1396 /* 1397 * Function to rep write 64 bit data into the PCI configuration space behind 1398 * the 21554's host interface. 1399 */ 1400 static void 1401 px_pci_config_rep_put64(ddi_acc_impl_t *handle, uint64_t *host_addr, 1402 uint64_t *dev_addr, size_t repcount, uint_t flags) 1403 { 1404 if (flags == DDI_DEV_AUTOINCR) 1405 for (; repcount; repcount--) 1406 px_pci_config_put64(handle, dev_addr++, *host_addr++); 1407 else 1408 for (; repcount; repcount--) 1409 px_pci_config_put64(handle, dev_addr, *host_addr++); 1410 } 1411 1412 /* 1413 * Provide a private access handle to route config access calls to Hypervisor. 1414 * Beware: Do all error checking for config space accesses before calling 1415 * this function. ie. do error checking from the calling function. 1416 * Due to a lack of meaningful error code in DDI, the gauranteed return of 1417 * DDI_SUCCESS from here makes the code organization readable/easier from 1418 * the generic code. 1419 */ 1420 /*ARGSUSED*/ 1421 int 1422 px_lib_map_vconfig(dev_info_t *dip, 1423 ddi_map_req_t *mp, pci_config_offset_t off, 1424 pci_regspec_t *rp, caddr_t *addrp) 1425 { 1426 int fmcap; 1427 ndi_err_t *errp; 1428 on_trap_data_t *otp; 1429 ddi_acc_hdl_t *hp; 1430 ddi_acc_impl_t *ap; 1431 uchar_t busnum; /* bus number */ 1432 uchar_t devnum; /* device number */ 1433 uchar_t funcnum; /* function number */ 1434 px_config_acc_pvt_t *px_pvt; 1435 1436 hp = (ddi_acc_hdl_t *)mp->map_handlep; 1437 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1438 1439 /* Check for mapping teardown operation */ 1440 if ((mp->map_op == DDI_MO_UNMAP) || 1441 (mp->map_op == DDI_MO_UNLOCK)) { 1442 /* free up memory allocated for the private access handle. */ 1443 px_pvt = (px_config_acc_pvt_t *)hp->ah_bus_private; 1444 kmem_free((void *)px_pvt, sizeof (px_config_acc_pvt_t)); 1445 1446 /* unmap operation of PCI IO/config space. */ 1447 return (DDI_SUCCESS); 1448 } 1449 1450 fmcap = ddi_fm_capable(dip); 1451 if (DDI_FM_ACC_ERR_CAP(fmcap)) { 1452 errp = ((ddi_acc_impl_t *)hp)->ahi_err; 1453 otp = (on_trap_data_t *)errp->err_ontrap; 1454 otp->ot_handle = (void *)(hp); 1455 otp->ot_prot = OT_DATA_ACCESS; 1456 errp->err_status = DDI_FM_OK; 1457 errp->err_expected = DDI_FM_ERR_UNEXPECTED; 1458 errp->err_cf = px_err_cfg_hdl_check; 1459 } 1460 1461 ap->ahi_get8 = px_pci_config_get8; 1462 ap->ahi_get16 = px_pci_config_get16; 1463 ap->ahi_get32 = px_pci_config_get32; 1464 ap->ahi_get64 = px_pci_config_get64; 1465 ap->ahi_put8 = px_pci_config_put8; 1466 ap->ahi_put16 = px_pci_config_put16; 1467 ap->ahi_put32 = px_pci_config_put32; 1468 ap->ahi_put64 = px_pci_config_put64; 1469 ap->ahi_rep_get8 = px_pci_config_rep_get8; 1470 ap->ahi_rep_get16 = px_pci_config_rep_get16; 1471 ap->ahi_rep_get32 = px_pci_config_rep_get32; 1472 ap->ahi_rep_get64 = px_pci_config_rep_get64; 1473 ap->ahi_rep_put8 = px_pci_config_rep_put8; 1474 ap->ahi_rep_put16 = px_pci_config_rep_put16; 1475 ap->ahi_rep_put32 = px_pci_config_rep_put32; 1476 ap->ahi_rep_put64 = px_pci_config_rep_put64; 1477 1478 /* Initialize to default check/notify functions */ 1479 ap->ahi_fault = 0; 1480 ap->ahi_fault_check = i_ddi_acc_fault_check; 1481 ap->ahi_fault_notify = i_ddi_acc_fault_notify; 1482 1483 /* allocate memory for our private handle */ 1484 px_pvt = (px_config_acc_pvt_t *) 1485 kmem_zalloc(sizeof (px_config_acc_pvt_t), KM_SLEEP); 1486 hp->ah_bus_private = (void *)px_pvt; 1487 1488 busnum = PCI_REG_BUS_G(rp->pci_phys_hi); 1489 devnum = PCI_REG_DEV_G(rp->pci_phys_hi); 1490 funcnum = PCI_REG_FUNC_G(rp->pci_phys_hi); 1491 1492 /* set up private data for use during IO routines */ 1493 1494 /* addr needed by the HV APIs */ 1495 px_pvt->raddr = busnum << 16 | devnum << 11 | funcnum << 8; 1496 /* 1497 * Address that specifies the actual offset into the 256MB 1498 * memory mapped configuration space, 4K per device. 1499 * First 12bits form the offset into 4K config space. 1500 * This address is only used during the IO routines to calculate 1501 * the offset at which the transaction must be performed. 1502 * Drivers bypassing DDI functions to access PCI config space will 1503 * panic the system since the following is a bogus virtual address. 1504 */ 1505 px_pvt->vaddr = busnum << 20 | devnum << 15 | funcnum << 12 | off; 1506 px_pvt->dip = dip; 1507 1508 DBG(DBG_LIB_CFG, dip, "px_config_setup: raddr 0x%x, vaddr 0x%x\n", 1509 px_pvt->raddr, px_pvt->vaddr); 1510 *addrp = (caddr_t)(uintptr_t)px_pvt->vaddr; 1511 return (DDI_SUCCESS); 1512 } 1513 1514 /*ARGSUSED*/ 1515 void 1516 px_lib_map_attr_check(ddi_map_req_t *mp) 1517 { 1518 } 1519 1520 /* 1521 * px_lib_log_safeacc_err: 1522 * Imitate a cpu/mem trap call when a peek/poke fails. 1523 * This will initiate something similar to px_fm_callback. 1524 */ 1525 static void 1526 px_lib_log_safeacc_err(px_t *px_p, ddi_acc_handle_t handle, int fme_flag, 1527 r_addr_t addr) 1528 { 1529 uint32_t addr_high, addr_low; 1530 pcie_req_id_t bdf; 1531 px_ranges_t *ranges_p; 1532 int range_len, i; 1533 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle; 1534 ddi_fm_error_t derr; 1535 1536 derr.fme_status = DDI_FM_NONFATAL; 1537 derr.fme_version = DDI_FME_VERSION; 1538 derr.fme_flag = fme_flag; 1539 derr.fme_ena = fm_ena_generate(0, FM_ENA_FMT1); 1540 derr.fme_acc_handle = handle; 1541 if (hp) 1542 hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED; 1543 1544 addr_high = (uint32_t)(addr >> 32); 1545 addr_low = (uint32_t)addr; 1546 1547 /* 1548 * Make sure this failed load came from this PCIe port. Check by 1549 * matching the upper 32 bits of the address with the ranges property. 1550 */ 1551 range_len = px_p->px_ranges_length / sizeof (px_ranges_t); 1552 i = 0; 1553 for (ranges_p = px_p->px_ranges_p; i < range_len; i++, ranges_p++) { 1554 if (ranges_p->parent_high == addr_high) { 1555 switch (ranges_p->child_high & PCI_ADDR_MASK) { 1556 case PCI_ADDR_CONFIG: 1557 bdf = (pcie_req_id_t)(addr_low >> 12); 1558 break; 1559 default: 1560 bdf = NULL; 1561 break; 1562 } 1563 break; 1564 } 1565 } 1566 1567 px_rp_en_q(px_p, bdf, addr, NULL); 1568 1569 if (px_fm_enter(px_p) == DDI_SUCCESS) { 1570 (void) px_scan_fabric(px_p, px_p->px_dip, &derr); 1571 px_fm_exit(px_p); 1572 } 1573 } 1574 1575 1576 #ifdef DEBUG 1577 int px_peekfault_cnt = 0; 1578 int px_pokefault_cnt = 0; 1579 #endif /* DEBUG */ 1580 1581 /* 1582 * Do a safe write to a device. 1583 * 1584 * When this function is given a handle (cautious access), all errors are 1585 * suppressed. 1586 * 1587 * When this function is not given a handle (poke), only Unsupported Request 1588 * and Completer Abort errors are suppressed. 1589 * 1590 * In all cases, all errors are returned in the function return status. 1591 */ 1592 1593 int 1594 px_lib_ctlops_poke(dev_info_t *dip, dev_info_t *rdip, 1595 peekpoke_ctlops_t *in_args) 1596 { 1597 px_t *px_p = DIP_TO_STATE(dip); 1598 px_pec_t *pec_p = px_p->px_pec_p; 1599 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle; 1600 1601 size_t repcount = in_args->repcount; 1602 size_t size = in_args->size; 1603 uintptr_t dev_addr = in_args->dev_addr; 1604 uintptr_t host_addr = in_args->host_addr; 1605 1606 int err = DDI_SUCCESS; 1607 uint64_t hvio_poke_status; 1608 uint32_t wrt_stat; 1609 1610 r_addr_t ra; 1611 uint64_t pokeval; 1612 pcie_req_id_t bdf; 1613 1614 ra = (r_addr_t)va_to_pa((void *)dev_addr); 1615 for (; repcount; repcount--) { 1616 1617 switch (size) { 1618 case sizeof (uint8_t): 1619 pokeval = *(uint8_t *)host_addr; 1620 break; 1621 case sizeof (uint16_t): 1622 pokeval = *(uint16_t *)host_addr; 1623 break; 1624 case sizeof (uint32_t): 1625 pokeval = *(uint32_t *)host_addr; 1626 break; 1627 case sizeof (uint64_t): 1628 pokeval = *(uint64_t *)host_addr; 1629 break; 1630 default: 1631 DBG(DBG_MAP, px_p->px_dip, 1632 "poke: invalid size %d passed\n", size); 1633 err = DDI_FAILURE; 1634 goto done; 1635 } 1636 1637 /* 1638 * Grab pokefault mutex since hypervisor does not guarantee 1639 * poke serialization. 1640 */ 1641 if (hp) { 1642 i_ndi_busop_access_enter(hp->ahi_common.ah_dip, 1643 (ddi_acc_handle_t)hp); 1644 pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED; 1645 } else { 1646 mutex_enter(&pec_p->pec_pokefault_mutex); 1647 pec_p->pec_safeacc_type = DDI_FM_ERR_POKE; 1648 } 1649 1650 if (pcie_get_bdf_from_dip(rdip, &bdf) != DDI_SUCCESS) { 1651 err = DDI_FAILURE; 1652 goto done; 1653 } 1654 1655 hvio_poke_status = hvio_poke(px_p->px_dev_hdl, ra, size, 1656 pokeval, bdf << 8, &wrt_stat); 1657 1658 if ((hvio_poke_status != H_EOK) || (wrt_stat != H_EOK)) { 1659 err = DDI_FAILURE; 1660 #ifdef DEBUG 1661 px_pokefault_cnt++; 1662 #endif 1663 /* 1664 * For CAUTIOUS and POKE access, notify FMA to 1665 * cleanup. Imitate a cpu/mem trap call like in sun4u. 1666 */ 1667 px_lib_log_safeacc_err(px_p, (ddi_acc_handle_t)hp, 1668 (hp ? DDI_FM_ERR_EXPECTED : 1669 DDI_FM_ERR_POKE), ra); 1670 1671 pec_p->pec_ontrap_data = NULL; 1672 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1673 if (hp) { 1674 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, 1675 (ddi_acc_handle_t)hp); 1676 } else { 1677 mutex_exit(&pec_p->pec_pokefault_mutex); 1678 } 1679 goto done; 1680 } 1681 1682 pec_p->pec_ontrap_data = NULL; 1683 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1684 if (hp) { 1685 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, 1686 (ddi_acc_handle_t)hp); 1687 } else { 1688 mutex_exit(&pec_p->pec_pokefault_mutex); 1689 } 1690 1691 host_addr += size; 1692 1693 if (in_args->flags == DDI_DEV_AUTOINCR) { 1694 dev_addr += size; 1695 ra = (r_addr_t)va_to_pa((void *)dev_addr); 1696 } 1697 } 1698 1699 done: 1700 return (err); 1701 } 1702 1703 1704 /*ARGSUSED*/ 1705 int 1706 px_lib_ctlops_peek(dev_info_t *dip, dev_info_t *rdip, 1707 peekpoke_ctlops_t *in_args, void *result) 1708 { 1709 px_t *px_p = DIP_TO_STATE(dip); 1710 px_pec_t *pec_p = px_p->px_pec_p; 1711 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle; 1712 1713 size_t repcount = in_args->repcount; 1714 uintptr_t dev_addr = in_args->dev_addr; 1715 uintptr_t host_addr = in_args->host_addr; 1716 1717 r_addr_t ra; 1718 uint32_t read_status; 1719 uint64_t hvio_peek_status; 1720 uint64_t peekval; 1721 int err = DDI_SUCCESS; 1722 1723 result = (void *)in_args->host_addr; 1724 1725 ra = (r_addr_t)va_to_pa((void *)dev_addr); 1726 for (; repcount; repcount--) { 1727 1728 /* Lock pokefault mutex so read doesn't mask a poke fault. */ 1729 if (hp) { 1730 i_ndi_busop_access_enter(hp->ahi_common.ah_dip, 1731 (ddi_acc_handle_t)hp); 1732 pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED; 1733 } else { 1734 mutex_enter(&pec_p->pec_pokefault_mutex); 1735 pec_p->pec_safeacc_type = DDI_FM_ERR_PEEK; 1736 } 1737 1738 hvio_peek_status = hvio_peek(px_p->px_dev_hdl, ra, 1739 in_args->size, &read_status, &peekval); 1740 1741 if ((hvio_peek_status != H_EOK) || (read_status != H_EOK)) { 1742 err = DDI_FAILURE; 1743 1744 /* 1745 * For CAUTIOUS and PEEK access, notify FMA to 1746 * cleanup. Imitate a cpu/mem trap call like in sun4u. 1747 */ 1748 px_lib_log_safeacc_err(px_p, (ddi_acc_handle_t)hp, 1749 (hp ? DDI_FM_ERR_EXPECTED : 1750 DDI_FM_ERR_PEEK), ra); 1751 1752 /* Stuff FFs in host addr if peek. */ 1753 if (hp == NULL) { 1754 int i; 1755 uint8_t *ff_addr = (uint8_t *)host_addr; 1756 for (i = 0; i < in_args->size; i++) 1757 *ff_addr++ = 0xff; 1758 } 1759 #ifdef DEBUG 1760 px_peekfault_cnt++; 1761 #endif 1762 pec_p->pec_ontrap_data = NULL; 1763 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1764 if (hp) { 1765 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, 1766 (ddi_acc_handle_t)hp); 1767 } else { 1768 mutex_exit(&pec_p->pec_pokefault_mutex); 1769 } 1770 goto done; 1771 1772 } 1773 pec_p->pec_ontrap_data = NULL; 1774 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1775 if (hp) { 1776 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, 1777 (ddi_acc_handle_t)hp); 1778 } else { 1779 mutex_exit(&pec_p->pec_pokefault_mutex); 1780 } 1781 1782 switch (in_args->size) { 1783 case sizeof (uint8_t): 1784 *(uint8_t *)host_addr = (uint8_t)peekval; 1785 break; 1786 case sizeof (uint16_t): 1787 *(uint16_t *)host_addr = (uint16_t)peekval; 1788 break; 1789 case sizeof (uint32_t): 1790 *(uint32_t *)host_addr = (uint32_t)peekval; 1791 break; 1792 case sizeof (uint64_t): 1793 *(uint64_t *)host_addr = (uint64_t)peekval; 1794 break; 1795 default: 1796 DBG(DBG_MAP, px_p->px_dip, 1797 "peek: invalid size %d passed\n", 1798 in_args->size); 1799 err = DDI_FAILURE; 1800 goto done; 1801 } 1802 1803 host_addr += in_args->size; 1804 1805 if (in_args->flags == DDI_DEV_AUTOINCR) { 1806 dev_addr += in_args->size; 1807 ra = (r_addr_t)va_to_pa((void *)dev_addr); 1808 } 1809 } 1810 done: 1811 return (err); 1812 } 1813 1814 1815 /* add interrupt vector */ 1816 int 1817 px_err_add_intr(px_fault_t *px_fault_p) 1818 { 1819 px_t *px_p = DIP_TO_STATE(px_fault_p->px_fh_dip); 1820 1821 DBG(DBG_LIB_INT, px_p->px_dip, 1822 "px_err_add_intr: calling add_ivintr"); 1823 1824 VERIFY(add_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL, 1825 (intrfunc)px_fault_p->px_err_func, (caddr_t)px_fault_p, NULL, 1826 (caddr_t)&px_fault_p->px_intr_payload[0]) == 0); 1827 1828 DBG(DBG_LIB_INT, px_p->px_dip, 1829 "px_err_add_intr: ib_intr_enable "); 1830 1831 px_ib_intr_enable(px_p, intr_dist_cpuid(), px_fault_p->px_intr_ino); 1832 1833 return (DDI_SUCCESS); 1834 } 1835 1836 /* remove interrupt vector */ 1837 void 1838 px_err_rem_intr(px_fault_t *px_fault_p) 1839 { 1840 px_t *px_p = DIP_TO_STATE(px_fault_p->px_fh_dip); 1841 1842 px_ib_intr_disable(px_p->px_ib_p, px_fault_p->px_intr_ino, 1843 IB_INTR_WAIT); 1844 1845 VERIFY(rem_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL) == 0); 1846 } 1847 1848 void 1849 px_cb_intr_redist(void *arg) 1850 { 1851 px_t *px_p = (px_t *)arg; 1852 px_ib_intr_dist_en(px_p->px_dip, intr_dist_cpuid(), 1853 px_p->px_inos[PX_INTR_XBC], B_FALSE); 1854 } 1855 1856 int 1857 px_cb_add_intr(px_fault_t *f_p) 1858 { 1859 px_t *px_p = DIP_TO_STATE(f_p->px_fh_dip); 1860 1861 DBG(DBG_LIB_INT, px_p->px_dip, 1862 "px_err_add_intr: calling add_ivintr"); 1863 1864 VERIFY(add_ivintr(f_p->px_fh_sysino, PX_ERR_PIL, 1865 (intrfunc)f_p->px_err_func, (caddr_t)f_p, NULL, 1866 (caddr_t)&f_p->px_intr_payload[0]) == 0); 1867 1868 intr_dist_add(px_cb_intr_redist, px_p); 1869 1870 DBG(DBG_LIB_INT, px_p->px_dip, 1871 "px_err_add_intr: ib_intr_enable "); 1872 1873 px_ib_intr_enable(px_p, intr_dist_cpuid(), f_p->px_intr_ino); 1874 1875 return (DDI_SUCCESS); 1876 } 1877 1878 void 1879 px_cb_rem_intr(px_fault_t *f_p) 1880 { 1881 intr_dist_rem(px_cb_intr_redist, DIP_TO_STATE(f_p->px_fh_dip)); 1882 px_err_rem_intr(f_p); 1883 } 1884 1885 #ifdef FMA 1886 void 1887 px_fill_rc_status(px_fault_t *px_fault_p, pciex_rc_error_regs_t *rc_status) 1888 { 1889 px_pec_err_t *err_pkt; 1890 1891 err_pkt = (px_pec_err_t *)px_fault_p->px_intr_payload; 1892 1893 /* initialise all the structure members */ 1894 rc_status->status_valid = 0; 1895 1896 if (err_pkt->pec_descr.P) { 1897 /* PCI Status Register */ 1898 rc_status->pci_err_status = err_pkt->pci_err_status; 1899 rc_status->status_valid |= PCI_ERR_STATUS_VALID; 1900 } 1901 1902 if (err_pkt->pec_descr.E) { 1903 /* PCIe Status Register */ 1904 rc_status->pcie_err_status = err_pkt->pcie_err_status; 1905 rc_status->status_valid |= PCIE_ERR_STATUS_VALID; 1906 } 1907 1908 if (err_pkt->pec_descr.U) { 1909 rc_status->ue_status = err_pkt->ue_reg_status; 1910 rc_status->status_valid |= UE_STATUS_VALID; 1911 } 1912 1913 if (err_pkt->pec_descr.H) { 1914 rc_status->ue_hdr1 = err_pkt->hdr[0]; 1915 rc_status->status_valid |= UE_HDR1_VALID; 1916 } 1917 1918 if (err_pkt->pec_descr.I) { 1919 rc_status->ue_hdr2 = err_pkt->hdr[1]; 1920 rc_status->status_valid |= UE_HDR2_VALID; 1921 } 1922 1923 /* ue_fst_err_ptr - not available for sun4v?? */ 1924 1925 1926 if (err_pkt->pec_descr.S) { 1927 rc_status->source_id = err_pkt->err_src_reg; 1928 rc_status->status_valid |= SOURCE_ID_VALID; 1929 } 1930 1931 if (err_pkt->pec_descr.R) { 1932 rc_status->root_err_status = err_pkt->root_err_status; 1933 rc_status->status_valid |= CE_STATUS_VALID; 1934 } 1935 } 1936 #endif 1937 1938 /*ARGSUSED*/ 1939 int 1940 px_lib_pmctl(int cmd, px_t *px_p) 1941 { 1942 return (DDI_FAILURE); 1943 } 1944 1945 /*ARGSUSED*/ 1946 uint_t 1947 px_pmeq_intr(caddr_t arg) 1948 { 1949 return (DDI_INTR_CLAIMED); 1950 } 1951 1952 /* 1953 * Unprotected raw reads/writes of fabric device's config space. 1954 * Only used for temporary PCI-E Fabric Error Handling. 1955 */ 1956 uint32_t 1957 px_fab_get(px_t *px_p, pcie_req_id_t bdf, uint16_t offset) { 1958 uint32_t data = 0; 1959 1960 (void) hvio_config_get(px_p->px_dev_hdl, 1961 (bdf << PX_RA_BDF_SHIFT), offset, 4, 1962 (pci_cfg_data_t *)&data); 1963 1964 return (data); 1965 } 1966 1967 void 1968 px_fab_set(px_t *px_p, pcie_req_id_t bdf, uint16_t offset, 1969 uint32_t val) { 1970 pci_cfg_data_t wdata = { 0 }; 1971 1972 wdata.qw = (uint32_t)val; 1973 (void) hvio_config_put(px_p->px_dev_hdl, 1974 (bdf << PX_RA_BDF_SHIFT), offset, 4, wdata); 1975 } 1976 1977 /*ARGSUSED*/ 1978 int 1979 px_lib_hotplug_init(dev_info_t *dip, void *arg) 1980 { 1981 return (DDI_ENOTSUP); 1982 } 1983 1984 /*ARGSUSED*/ 1985 void 1986 px_lib_hotplug_uninit(dev_info_t *dip) 1987 { 1988 } 1989 1990 /*ARGSUSED*/ 1991 void 1992 px_hp_intr_redist(px_t *px_p) 1993 { 1994 } 1995 1996 /* Dummy cpr add callback */ 1997 /*ARGSUSED*/ 1998 void 1999 px_cpr_add_callb(px_t *px_p) 2000 { 2001 } 2002 2003 /* Dummy cpr rem callback */ 2004 /*ARGSUSED*/ 2005 void 2006 px_cpr_rem_callb(px_t *px_p) 2007 { 2008 } 2009 2010 /*ARGSUSED*/ 2011 boolean_t 2012 px_lib_is_in_drain_state(px_t *px_p) 2013 { 2014 return (B_FALSE); 2015 } 2016 2017 /* 2018 * There is no IOAPI to get the BDF of the pcie root port nexus at this moment. 2019 * Assume it is 0x0000, until otherwise noted. For now, all sun4v platforms 2020 * have programmed the BDF to be 0x0000. 2021 */ 2022 /*ARGSUSED*/ 2023 pcie_req_id_t 2024 px_lib_get_bdf(px_t *px_p) 2025 { 2026 return (0x0000); 2027 } 2028 2029 int 2030 px_lib_get_root_complex_mps(px_t *px_p, dev_info_t *dip, int *mps) 2031 { 2032 pci_device_t bdf = px_lib_get_bdf(px_p); 2033 2034 if (hvio_get_rp_mps_cap(DIP_TO_HANDLE(dip), bdf, mps) == H_EOK) 2035 return (DDI_SUCCESS); 2036 else 2037 return (DDI_FAILURE); 2038 } 2039 2040 int 2041 px_lib_set_root_complex_mps(px_t *px_p, dev_info_t *dip, int mps) 2042 { 2043 pci_device_t bdf = px_lib_get_bdf(px_p); 2044 2045 if (hvio_set_rp_mps(DIP_TO_HANDLE(dip), bdf, mps) == H_EOK) 2046 return (DDI_SUCCESS); 2047 else 2048 return (DDI_FAILURE); 2049 } 2050