1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 #include <sys/types.h> 26 #include <sys/sysmacros.h> 27 #include <sys/ddi.h> 28 #include <sys/async.h> 29 #include <sys/sunddi.h> 30 #include <sys/ddifm.h> 31 #include <sys/fm/protocol.h> 32 #include <sys/vmem.h> 33 #include <sys/intr.h> 34 #include <sys/ivintr.h> 35 #include <sys/errno.h> 36 #include <sys/hypervisor_api.h> 37 #include <sys/hsvc.h> 38 #include <px_obj.h> 39 #include <sys/machsystm.h> 40 #include <sys/sunndi.h> 41 #include <sys/pcie_impl.h> 42 #include "px_lib4v.h" 43 #include "px_err.h" 44 #include <sys/pci_cfgacc.h> 45 #include <sys/pci_cfgacc_4v.h> 46 47 48 /* mask for the ranges property in calculating the real PFN range */ 49 uint_t px_ranges_phi_mask = ((1 << 28) -1); 50 51 /* 52 * Hypervisor VPCI services information for the px nexus driver. 53 */ 54 static uint64_t px_vpci_maj_ver; /* Negotiated VPCI API major version */ 55 static uint64_t px_vpci_min_ver; /* Negotiated VPCI API minor version */ 56 static uint_t px_vpci_users = 0; /* VPCI API users */ 57 static hsvc_info_t px_hsvc_vpci = { 58 HSVC_REV_1, NULL, HSVC_GROUP_VPCI, PX_VPCI_MAJOR_VER, 59 PX_VPCI_MINOR_VER, "PX" 60 }; 61 62 /* 63 * Hypervisor SDIO services information for the px nexus driver. 64 */ 65 static uint64_t px_sdio_min_ver; /* Negotiated SDIO API minor version */ 66 static uint_t px_sdio_users = 0; /* SDIO API users */ 67 static hsvc_info_t px_hsvc_sdio = { 68 HSVC_REV_1, NULL, HSVC_GROUP_SDIO, PX_SDIO_MAJOR_VER, 69 PX_SDIO_MINOR_VER, "PX" 70 }; 71 72 /* 73 * Hypervisor SDIO ERR services information for the px nexus driver. 74 */ 75 static uint64_t px_sdio_err_min_ver; /* Negotiated SDIO ERR API */ 76 /* minor version */ 77 static uint_t px_sdio_err_users = 0; /* SDIO ERR API users */ 78 static hsvc_info_t px_hsvc_sdio_err = { 79 HSVC_REV_1, NULL, HSVC_GROUP_SDIO_ERR, PX_SDIO_ERR_MAJOR_VER, 80 PX_SDIO_ERR_MINOR_VER, "PX" 81 }; 82 83 #define CHILD_LOANED "child_loaned" 84 static int px_lib_count_waiting_dev(dev_info_t *); 85 86 int 87 px_lib_dev_init(dev_info_t *dip, devhandle_t *dev_hdl) 88 { 89 px_nexus_regspec_t *rp; 90 uint_t reglen; 91 int ret; 92 px_t *px_p = DIP_TO_STATE(dip); 93 uint64_t mjrnum; 94 uint64_t mnrnum; 95 96 DBG(DBG_ATTACH, dip, "px_lib_dev_init: dip 0x%p\n", dip); 97 98 /* 99 * Check HV intr group api versioning. 100 * This driver uses the old interrupt routines which are supported 101 * in old firmware in the CORE API group and in newer firmware in 102 * the INTR API group. Support for these calls will be dropped 103 * once the INTR API group major goes to 2. 104 */ 105 if ((hsvc_version(HSVC_GROUP_INTR, &mjrnum, &mnrnum) == 0) && 106 (mjrnum > 1)) { 107 cmn_err(CE_WARN, "px: unsupported intr api group: " 108 "maj:0x%lx, min:0x%lx", mjrnum, mnrnum); 109 return (ENOTSUP); 110 } 111 112 ret = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 113 "reg", (uchar_t **)&rp, ®len); 114 if (ret != DDI_PROP_SUCCESS) { 115 DBG(DBG_ATTACH, dip, "px_lib_dev_init failed ret=%d\n", ret); 116 return (DDI_FAILURE); 117 } 118 119 /* 120 * Initilize device handle. The device handle uniquely identifies 121 * a SUN4V device. It consists of the lower 28-bits of the hi-cell 122 * of the first entry of the SUN4V device's "reg" property as 123 * defined by the SUN4V Bus Binding to Open Firmware. 124 */ 125 *dev_hdl = (devhandle_t)((rp->phys_addr >> 32) & DEVHDLE_MASK); 126 ddi_prop_free(rp); 127 128 /* 129 * hotplug implementation requires this property to be associated with 130 * any indirect PCI config access services 131 */ 132 (void) ddi_prop_update_int(makedevice(ddi_driver_major(dip), 133 PCI_MINOR_NUM(ddi_get_instance(dip), PCI_DEVCTL_MINOR)), dip, 134 PCI_BUS_CONF_MAP_PROP, 1); 135 136 DBG(DBG_ATTACH, dip, "px_lib_dev_init: dev_hdl 0x%llx\n", *dev_hdl); 137 138 /* 139 * If a /pci node has a pci-intx-not-supported property, this property 140 * represents that the fabric doesn't support fixed interrupt. 141 */ 142 if (!ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 143 "pci-intx-not-supported")) { 144 DBG(DBG_ATTACH, dip, "px_lib_dev_init: " 145 "pci-intx-not-supported is not found, dip=0x%p\n", dip); 146 px_p->px_supp_intr_types |= DDI_INTR_TYPE_FIXED; 147 } 148 149 /* 150 * Negotiate the API version for VPCI hypervisor services. 151 */ 152 if (px_vpci_users == 0) { 153 if ((ret = hsvc_register(&px_hsvc_vpci, &px_vpci_min_ver)) 154 == 0) { 155 px_vpci_maj_ver = px_hsvc_vpci.hsvc_major; 156 goto hv_negotiation_complete; 157 } 158 /* 159 * Negotiation with the latest known VPCI hypervisor services 160 * failed. Fallback to version 1.0. 161 */ 162 px_hsvc_vpci.hsvc_major = PX_HSVC_MAJOR_VER_1; 163 px_hsvc_vpci.hsvc_minor = PX_HSVC_MINOR_VER_0; 164 165 if ((ret = hsvc_register(&px_hsvc_vpci, &px_vpci_min_ver)) 166 == 0) { 167 px_vpci_maj_ver = px_hsvc_vpci.hsvc_major; 168 goto hv_negotiation_complete; 169 } 170 171 cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services " 172 "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d\n", 173 px_hsvc_vpci.hsvc_modname, px_hsvc_vpci.hsvc_group, 174 px_hsvc_vpci.hsvc_major, px_hsvc_vpci.hsvc_minor, ret); 175 176 return (DDI_FAILURE); 177 } 178 hv_negotiation_complete: 179 180 px_vpci_users++; 181 182 DBG(DBG_ATTACH, dip, "px_lib_dev_init: negotiated VPCI API version, " 183 "major 0x%lx minor 0x%lx\n", px_vpci_maj_ver, 184 px_vpci_min_ver); 185 186 /* 187 * Negotiate the API version for SDIO hypervisor services. 188 */ 189 if ((px_sdio_users == 0) && 190 ((ret = hsvc_register(&px_hsvc_sdio, &px_sdio_min_ver)) != 0)) { 191 DBG(DBG_ATTACH, dip, "%s: cannot negotiate hypervisor " 192 "services group: 0x%lx major: 0x%lx minor: 0x%lx " 193 "errno: %d\n", px_hsvc_sdio.hsvc_modname, 194 px_hsvc_sdio.hsvc_group, px_hsvc_sdio.hsvc_major, 195 px_hsvc_sdio.hsvc_minor, ret); 196 } else { 197 px_sdio_users++; 198 DBG(DBG_ATTACH, dip, "px_lib_dev_init: negotiated SDIO API" 199 "version, major 0x%lx minor 0x%lx\n", 200 px_hsvc_sdio.hsvc_major, px_sdio_min_ver); 201 } 202 203 /* 204 * Negotiate the API version for SDIO ERR hypervisor services. 205 */ 206 if ((px_sdio_err_users == 0) && 207 ((ret = hsvc_register(&px_hsvc_sdio_err, 208 &px_sdio_err_min_ver)) != 0)) { 209 DBG(DBG_ATTACH, dip, "%s: cannot negotiate SDIO ERR hypervisor " 210 "services group: 0x%lx major: 0x%lx minor: 0x%lx " 211 "errno: %d\n", px_hsvc_sdio_err.hsvc_modname, 212 px_hsvc_sdio_err.hsvc_group, px_hsvc_sdio_err.hsvc_major, 213 px_hsvc_sdio_err.hsvc_minor, ret); 214 } else { 215 px_sdio_err_users++; 216 DBG(DBG_ATTACH, dip, "px_lib_dev_init: negotiated SDIO ERR API " 217 "version, major 0x%lx minor 0x%lx\n", 218 px_hsvc_sdio_err.hsvc_major, px_sdio_err_min_ver); 219 } 220 221 /* 222 * Find out the number of dev we need to wait under this RC 223 * before we issue fabric sync hypercall 224 */ 225 px_p->px_plat_p = (void *)(uintptr_t)px_lib_count_waiting_dev(dip); 226 DBG(DBG_ATTACH, dip, "Found %d bridges need waiting under RC %p", 227 (int)(uintptr_t)px_p->px_plat_p, dip); 228 return (DDI_SUCCESS); 229 } 230 231 /*ARGSUSED*/ 232 int 233 px_lib_dev_fini(dev_info_t *dip) 234 { 235 DBG(DBG_DETACH, dip, "px_lib_dev_fini: dip 0x%p\n", dip); 236 237 (void) ddi_prop_remove(makedevice(ddi_driver_major(dip), 238 PCI_MINOR_NUM(ddi_get_instance(dip), PCI_DEVCTL_MINOR)), dip, 239 PCI_BUS_CONF_MAP_PROP); 240 241 if (--px_vpci_users == 0) 242 (void) hsvc_unregister(&px_hsvc_vpci); 243 244 if (--px_sdio_users == 0) 245 (void) hsvc_unregister(&px_hsvc_sdio); 246 247 if (--px_sdio_err_users == 0) 248 (void) hsvc_unregister(&px_hsvc_sdio_err); 249 250 return (DDI_SUCCESS); 251 } 252 253 /*ARGSUSED*/ 254 int 255 px_lib_intr_devino_to_sysino(dev_info_t *dip, devino_t devino, 256 sysino_t *sysino) 257 { 258 uint64_t ret; 259 260 DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: dip 0x%p " 261 "devino 0x%x\n", dip, devino); 262 263 if ((ret = hvio_intr_devino_to_sysino(DIP_TO_HANDLE(dip), 264 devino, sysino)) != H_EOK) { 265 DBG(DBG_LIB_INT, dip, 266 "hvio_intr_devino_to_sysino failed, ret 0x%lx\n", ret); 267 return (DDI_FAILURE); 268 } 269 270 DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: sysino 0x%llx\n", 271 *sysino); 272 273 return (DDI_SUCCESS); 274 } 275 276 /*ARGSUSED*/ 277 int 278 px_lib_intr_getvalid(dev_info_t *dip, sysino_t sysino, 279 intr_valid_state_t *intr_valid_state) 280 { 281 uint64_t ret; 282 283 DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: dip 0x%p sysino 0x%llx\n", 284 dip, sysino); 285 286 if ((ret = hvio_intr_getvalid(sysino, 287 (int *)intr_valid_state)) != H_EOK) { 288 DBG(DBG_LIB_INT, dip, "hvio_intr_getvalid failed, ret 0x%lx\n", 289 ret); 290 return (DDI_FAILURE); 291 } 292 293 DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: intr_valid_state 0x%x\n", 294 *intr_valid_state); 295 296 return (DDI_SUCCESS); 297 } 298 299 /*ARGSUSED*/ 300 int 301 px_lib_intr_setvalid(dev_info_t *dip, sysino_t sysino, 302 intr_valid_state_t intr_valid_state) 303 { 304 uint64_t ret; 305 306 DBG(DBG_LIB_INT, dip, "px_lib_intr_setvalid: dip 0x%p sysino 0x%llx " 307 "intr_valid_state 0x%x\n", dip, sysino, intr_valid_state); 308 309 if ((ret = hvio_intr_setvalid(sysino, intr_valid_state)) != H_EOK) { 310 DBG(DBG_LIB_INT, dip, "hvio_intr_setvalid failed, ret 0x%lx\n", 311 ret); 312 return (DDI_FAILURE); 313 } 314 315 return (DDI_SUCCESS); 316 } 317 318 /*ARGSUSED*/ 319 int 320 px_lib_intr_getstate(dev_info_t *dip, sysino_t sysino, 321 intr_state_t *intr_state) 322 { 323 uint64_t ret; 324 325 DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: dip 0x%p sysino 0x%llx\n", 326 dip, sysino); 327 328 if ((ret = hvio_intr_getstate(sysino, (int *)intr_state)) != H_EOK) { 329 DBG(DBG_LIB_INT, dip, "hvio_intr_getstate failed, ret 0x%lx\n", 330 ret); 331 return (DDI_FAILURE); 332 } 333 334 DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: intr_state 0x%x\n", 335 *intr_state); 336 337 return (DDI_SUCCESS); 338 } 339 340 /*ARGSUSED*/ 341 int 342 px_lib_intr_setstate(dev_info_t *dip, sysino_t sysino, 343 intr_state_t intr_state) 344 { 345 uint64_t ret; 346 347 DBG(DBG_LIB_INT, dip, "px_lib_intr_setstate: dip 0x%p sysino 0x%llx " 348 "intr_state 0x%x\n", dip, sysino, intr_state); 349 350 if ((ret = hvio_intr_setstate(sysino, intr_state)) != H_EOK) { 351 DBG(DBG_LIB_INT, dip, "hvio_intr_setstate failed, ret 0x%lx\n", 352 ret); 353 return (DDI_FAILURE); 354 } 355 356 return (DDI_SUCCESS); 357 } 358 359 /*ARGSUSED*/ 360 int 361 px_lib_intr_gettarget(dev_info_t *dip, sysino_t sysino, cpuid_t *cpuid) 362 { 363 uint64_t ret; 364 365 DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: dip 0x%p sysino 0x%llx\n", 366 dip, sysino); 367 368 if ((ret = hvio_intr_gettarget(sysino, cpuid)) != H_EOK) { 369 DBG(DBG_LIB_INT, dip, 370 "hvio_intr_gettarget failed, ret 0x%lx\n", ret); 371 return (DDI_FAILURE); 372 } 373 374 DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: cpuid 0x%x\n", *cpuid); 375 376 return (DDI_SUCCESS); 377 } 378 379 /*ARGSUSED*/ 380 int 381 px_lib_intr_settarget(dev_info_t *dip, sysino_t sysino, cpuid_t cpuid) 382 { 383 uint64_t ret; 384 385 DBG(DBG_LIB_INT, dip, "px_lib_intr_settarget: dip 0x%p sysino 0x%llx " 386 "cpuid 0x%x\n", dip, sysino, cpuid); 387 388 ret = hvio_intr_settarget(sysino, cpuid); 389 if (ret == H_ECPUERROR) { 390 cmn_err(CE_PANIC, 391 "px_lib_intr_settarget: hvio_intr_settarget failed, " 392 "ret = 0x%lx, cpuid = 0x%x, sysino = 0x%lx\n", ret, 393 cpuid, sysino); 394 } else if (ret != H_EOK) { 395 DBG(DBG_LIB_INT, dip, 396 "hvio_intr_settarget failed, ret 0x%lx\n", ret); 397 return (DDI_FAILURE); 398 } 399 400 return (DDI_SUCCESS); 401 } 402 403 /*ARGSUSED*/ 404 int 405 px_lib_intr_reset(dev_info_t *dip) 406 { 407 px_t *px_p = DIP_TO_STATE(dip); 408 px_ib_t *ib_p = px_p->px_ib_p; 409 px_ino_t *ino_p; 410 411 DBG(DBG_LIB_INT, dip, "px_lib_intr_reset: dip 0x%p\n", dip); 412 413 mutex_enter(&ib_p->ib_ino_lst_mutex); 414 415 /* Reset all Interrupts */ 416 for (ino_p = ib_p->ib_ino_lst; ino_p; ino_p = ino_p->ino_next_p) { 417 if (px_lib_intr_setstate(dip, ino_p->ino_sysino, 418 INTR_IDLE_STATE) != DDI_SUCCESS) 419 return (BF_FATAL); 420 } 421 422 mutex_exit(&ib_p->ib_ino_lst_mutex); 423 424 return (BF_NONE); 425 } 426 427 /*ARGSUSED*/ 428 int 429 px_lib_iommu_map(dev_info_t *dip, tsbid_t tsbid, pages_t pages, 430 io_attributes_t attr, void *addr, size_t pfn_index, int flags) 431 { 432 tsbnum_t tsb_num = PCI_TSBID_TO_TSBNUM(tsbid); 433 tsbindex_t tsb_index = PCI_TSBID_TO_TSBINDEX(tsbid); 434 io_page_list_t *pfns, *pfn_p; 435 pages_t ttes_mapped = 0; 436 int i, err = DDI_SUCCESS; 437 438 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: dip 0x%p tsbid 0x%llx " 439 "pages 0x%x attr 0x%llx addr 0x%p pfn_index 0x%llx flags 0x%x\n", 440 dip, tsbid, pages, attr, addr, pfn_index, flags); 441 442 if ((pfns = pfn_p = kmem_zalloc((pages * sizeof (io_page_list_t)), 443 KM_NOSLEEP)) == NULL) { 444 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: kmem_zalloc failed\n"); 445 return (DDI_FAILURE); 446 } 447 448 for (i = 0; i < pages; i++) 449 pfns[i] = MMU_PTOB(PX_ADDR2PFN(addr, pfn_index, flags, i)); 450 451 /* 452 * If HV VPCI version is 2.0 and higher, pass BDF, phantom function, 453 * and relaxed ordering attributes. Otherwise, pass only read or write 454 * attribute. 455 */ 456 if ((px_vpci_maj_ver == PX_HSVC_MAJOR_VER_1) && 457 (px_vpci_min_ver == PX_HSVC_MINOR_VER_0)) 458 attr = attr & (PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE); 459 460 while ((ttes_mapped = pfn_p - pfns) < pages) { 461 uintptr_t ra = va_to_pa(pfn_p); 462 pages_t ttes2map; 463 uint64_t ret; 464 465 ttes2map = (MMU_PAGE_SIZE - P2PHASE(ra, MMU_PAGE_SIZE)) >> 3; 466 ra = MMU_PTOB(MMU_BTOP(ra)); 467 468 for (ttes2map = MIN(ttes2map, pages - ttes_mapped); ttes2map; 469 ttes2map -= ttes_mapped, pfn_p += ttes_mapped) { 470 471 ttes_mapped = 0; 472 if ((ret = hvio_iommu_map(DIP_TO_HANDLE(dip), 473 PCI_TSBID(tsb_num, tsb_index + (pfn_p - pfns)), 474 ttes2map, attr, (io_page_list_t *)(ra | 475 ((uintptr_t)pfn_p & MMU_PAGE_OFFSET)), 476 &ttes_mapped)) != H_EOK) { 477 DBG(DBG_LIB_DMA, dip, "hvio_iommu_map failed " 478 "ret 0x%lx\n", ret); 479 480 ttes_mapped = pfn_p - pfns; 481 err = DDI_FAILURE; 482 goto cleanup; 483 } 484 485 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: tsb_num 0x%x " 486 "tsb_index 0x%lx ttes_to_map 0x%lx attr 0x%llx " 487 "ra 0x%p ttes_mapped 0x%x\n", tsb_num, 488 tsb_index + (pfn_p - pfns), ttes2map, attr, 489 ra | ((uintptr_t)pfn_p & MMU_PAGE_OFFSET), 490 ttes_mapped); 491 } 492 } 493 494 cleanup: 495 if ((err == DDI_FAILURE) && ttes_mapped) 496 (void) px_lib_iommu_demap(dip, tsbid, ttes_mapped); 497 498 kmem_free(pfns, pages * sizeof (io_page_list_t)); 499 return (err); 500 } 501 502 /*ARGSUSED*/ 503 int 504 px_lib_iommu_demap(dev_info_t *dip, tsbid_t tsbid, pages_t pages) 505 { 506 tsbnum_t tsb_num = PCI_TSBID_TO_TSBNUM(tsbid); 507 tsbindex_t tsb_index = PCI_TSBID_TO_TSBINDEX(tsbid); 508 pages_t ttes2demap, ttes_demapped = 0; 509 uint64_t ret; 510 511 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: dip 0x%p tsbid 0x%llx " 512 "pages 0x%x\n", dip, tsbid, pages); 513 514 for (ttes2demap = pages; ttes2demap; 515 ttes2demap -= ttes_demapped, tsb_index += ttes_demapped) { 516 if ((ret = hvio_iommu_demap(DIP_TO_HANDLE(dip), 517 PCI_TSBID(tsb_num, tsb_index), ttes2demap, 518 &ttes_demapped)) != H_EOK) { 519 DBG(DBG_LIB_DMA, dip, "hvio_iommu_demap failed, " 520 "ret 0x%lx\n", ret); 521 522 return (DDI_FAILURE); 523 } 524 525 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: tsb_num 0x%x " 526 "tsb_index 0x%lx ttes_to_demap 0x%lx ttes_demapped 0x%x\n", 527 tsb_num, tsb_index, ttes2demap, ttes_demapped); 528 } 529 530 return (DDI_SUCCESS); 531 } 532 533 /*ARGSUSED*/ 534 int 535 px_lib_iommu_getmap(dev_info_t *dip, tsbid_t tsbid, io_attributes_t *attr_p, 536 r_addr_t *r_addr_p) 537 { 538 uint64_t ret; 539 540 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: dip 0x%p tsbid 0x%llx\n", 541 dip, tsbid); 542 543 if ((ret = hvio_iommu_getmap(DIP_TO_HANDLE(dip), tsbid, 544 attr_p, r_addr_p)) != H_EOK) { 545 DBG(DBG_LIB_DMA, dip, 546 "hvio_iommu_getmap failed, ret 0x%lx\n", ret); 547 548 return ((ret == H_ENOMAP) ? DDI_DMA_NOMAPPING:DDI_FAILURE); 549 } 550 551 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: attr 0x%llx " 552 "r_addr 0x%llx\n", *attr_p, *r_addr_p); 553 554 return (DDI_SUCCESS); 555 } 556 557 /*ARGSUSED*/ 558 int 559 px_lib_iommu_detach(px_t *px_p) 560 { 561 return (DDI_SUCCESS); 562 } 563 564 /*ARGSUSED*/ 565 uint64_t 566 px_get_rng_parent_hi_mask(px_t *px_p) 567 { 568 return (PX_RANGE_PROP_MASK); 569 } 570 571 /* 572 * Checks dma attributes against system bypass ranges 573 * A sun4v device must be capable of generating the entire 64-bit 574 * address in order to perform bypass DMA. 575 */ 576 /*ARGSUSED*/ 577 int 578 px_lib_dma_bypass_rngchk(dev_info_t *dip, ddi_dma_attr_t *attr_p, 579 uint64_t *lo_p, uint64_t *hi_p) 580 { 581 if ((attr_p->dma_attr_addr_lo != 0ull) || 582 (attr_p->dma_attr_addr_hi != UINT64_MAX)) { 583 584 return (DDI_DMA_BADATTR); 585 } 586 587 *lo_p = 0ull; 588 *hi_p = UINT64_MAX; 589 590 return (DDI_SUCCESS); 591 } 592 593 594 /*ARGSUSED*/ 595 int 596 px_lib_iommu_getbypass(dev_info_t *dip, r_addr_t ra, io_attributes_t attr, 597 io_addr_t *io_addr_p) 598 { 599 uint64_t ret; 600 601 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: dip 0x%p ra 0x%llx " 602 "attr 0x%llx\n", dip, ra, attr); 603 /* 604 * If HV VPCI version is 2.0 and higher, pass BDF, phantom function, 605 * and relaxed ordering attributes. Otherwise, pass only read or write 606 * attribute. 607 */ 608 if ((px_vpci_maj_ver == PX_HSVC_MAJOR_VER_1) && 609 (px_vpci_min_ver == PX_HSVC_MINOR_VER_0)) 610 attr &= PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE; 611 612 if ((ret = hvio_iommu_getbypass(DIP_TO_HANDLE(dip), ra, 613 attr, io_addr_p)) != H_EOK) { 614 DBG(DBG_LIB_DMA, dip, 615 "hvio_iommu_getbypass failed, ret 0x%lx\n", ret); 616 return (ret == H_ENOTSUPPORTED ? DDI_ENOTSUP : DDI_FAILURE); 617 } 618 619 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: io_addr 0x%llx\n", 620 *io_addr_p); 621 622 return (DDI_SUCCESS); 623 } 624 625 /* 626 * Returns any needed IO address bit(s) for relaxed ordering in IOMMU 627 * bypass mode. 628 */ 629 /* ARGSUSED */ 630 uint64_t 631 px_lib_ro_bypass(dev_info_t *dip, io_attributes_t attr, uint64_t ioaddr) 632 { 633 return (ioaddr); 634 } 635 636 /*ARGSUSED*/ 637 int 638 px_lib_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 639 off_t off, size_t len, uint_t cache_flags) 640 { 641 ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; 642 uint64_t sync_dir; 643 size_t bytes_synced; 644 int end, idx; 645 off_t pg_off; 646 devhandle_t hdl = DIP_TO_HANDLE(dip); /* need to cache hdl */ 647 648 DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: dip 0x%p rdip 0x%p " 649 "handle 0x%llx off 0x%x len 0x%x flags 0x%x\n", 650 dip, rdip, handle, off, len, cache_flags); 651 652 if (!(mp->dmai_flags & PX_DMAI_FLAGS_INUSE)) { 653 cmn_err(CE_WARN, "%s%d: Unbound dma handle %p.", 654 ddi_driver_name(rdip), ddi_get_instance(rdip), (void *)mp); 655 return (DDI_FAILURE); 656 } 657 658 if (mp->dmai_flags & PX_DMAI_FLAGS_NOSYNC) 659 return (DDI_SUCCESS); 660 661 if (!len) 662 len = mp->dmai_size; 663 664 if (mp->dmai_rflags & DDI_DMA_READ) 665 sync_dir = HVIO_DMA_SYNC_DIR_FROM_DEV; 666 else 667 sync_dir = HVIO_DMA_SYNC_DIR_TO_DEV; 668 669 off += mp->dmai_offset; 670 pg_off = off & MMU_PAGEOFFSET; 671 672 DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: page offset %x size %x\n", 673 pg_off, len); 674 675 /* sync on page basis */ 676 end = MMU_BTOPR(off + len - 1); 677 for (idx = MMU_BTOP(off); idx < end; idx++, 678 len -= bytes_synced, pg_off = 0) { 679 size_t bytes_to_sync = bytes_to_sync = 680 MIN(len, MMU_PAGESIZE - pg_off); 681 682 if (hvio_dma_sync(hdl, MMU_PTOB(PX_GET_MP_PFN(mp, idx)) + 683 pg_off, bytes_to_sync, sync_dir, &bytes_synced) != H_EOK) 684 break; 685 686 DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: Called hvio_dma_sync " 687 "ra = %p bytes to sync = %x bytes synced %x\n", 688 MMU_PTOB(PX_GET_MP_PFN(mp, idx)) + pg_off, bytes_to_sync, 689 bytes_synced); 690 691 if (bytes_to_sync != bytes_synced) 692 break; 693 } 694 695 return (len ? DDI_FAILURE : DDI_SUCCESS); 696 } 697 698 699 /* 700 * MSIQ Functions: 701 */ 702 703 /*ARGSUSED*/ 704 int 705 px_lib_msiq_init(dev_info_t *dip) 706 { 707 px_t *px_p = DIP_TO_STATE(dip); 708 px_msiq_state_t *msiq_state_p = &px_p->px_ib_p->ib_msiq_state; 709 r_addr_t ra; 710 size_t msiq_size; 711 uint_t rec_cnt; 712 int i, err = DDI_SUCCESS; 713 uint64_t ret; 714 715 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_init: dip 0x%p\n", dip); 716 717 msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t); 718 719 /* sun4v requires all EQ allocation to be on q size boundary */ 720 if ((msiq_state_p->msiq_buf_p = contig_mem_alloc_align( 721 msiq_state_p->msiq_cnt * msiq_size, msiq_size)) == NULL) { 722 DBG(DBG_LIB_MSIQ, dip, 723 "px_lib_msiq_init: Contig alloc failed\n"); 724 725 return (DDI_FAILURE); 726 } 727 728 for (i = 0; i < msiq_state_p->msiq_cnt; i++) { 729 msiq_state_p->msiq_p[i].msiq_base_p = (msiqhead_t *) 730 ((caddr_t)msiq_state_p->msiq_buf_p + (i * msiq_size)); 731 732 ra = (r_addr_t)va_to_pa((caddr_t)msiq_state_p->msiq_buf_p + 733 (i * msiq_size)); 734 735 if ((ret = hvio_msiq_conf(DIP_TO_HANDLE(dip), 736 (i + msiq_state_p->msiq_1st_msiq_id), 737 ra, msiq_state_p->msiq_rec_cnt)) != H_EOK) { 738 DBG(DBG_LIB_MSIQ, dip, 739 "hvio_msiq_conf failed, ret 0x%lx\n", ret); 740 err = DDI_FAILURE; 741 break; 742 } 743 744 if ((err = px_lib_msiq_info(dip, 745 (i + msiq_state_p->msiq_1st_msiq_id), 746 &ra, &rec_cnt)) != DDI_SUCCESS) { 747 DBG(DBG_LIB_MSIQ, dip, 748 "px_lib_msiq_info failed, ret 0x%x\n", err); 749 err = DDI_FAILURE; 750 break; 751 } 752 753 DBG(DBG_LIB_MSIQ, dip, 754 "px_lib_msiq_init: ra 0x%p rec_cnt 0x%x\n", ra, rec_cnt); 755 } 756 757 return (err); 758 } 759 760 /*ARGSUSED*/ 761 int 762 px_lib_msiq_fini(dev_info_t *dip) 763 { 764 px_t *px_p = DIP_TO_STATE(dip); 765 px_msiq_state_t *msiq_state_p = &px_p->px_ib_p->ib_msiq_state; 766 size_t msiq_size; 767 768 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_fini: dip 0x%p\n", dip); 769 msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t); 770 771 if (msiq_state_p->msiq_buf_p != NULL) 772 contig_mem_free(msiq_state_p->msiq_buf_p, 773 msiq_state_p->msiq_cnt * msiq_size); 774 775 return (DDI_SUCCESS); 776 } 777 778 /*ARGSUSED*/ 779 int 780 px_lib_msiq_info(dev_info_t *dip, msiqid_t msiq_id, r_addr_t *ra_p, 781 uint_t *msiq_rec_cnt_p) 782 { 783 uint64_t ret; 784 785 DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: dip 0x%p msiq_id 0x%x\n", 786 dip, msiq_id); 787 788 if ((ret = hvio_msiq_info(DIP_TO_HANDLE(dip), 789 msiq_id, ra_p, msiq_rec_cnt_p)) != H_EOK) { 790 DBG(DBG_LIB_MSIQ, dip, 791 "hvio_msiq_info failed, ret 0x%lx\n", ret); 792 return (DDI_FAILURE); 793 } 794 795 DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: ra_p 0x%p msiq_rec_cnt 0x%x\n", 796 ra_p, *msiq_rec_cnt_p); 797 798 return (DDI_SUCCESS); 799 } 800 801 /*ARGSUSED*/ 802 int 803 px_lib_msiq_getvalid(dev_info_t *dip, msiqid_t msiq_id, 804 pci_msiq_valid_state_t *msiq_valid_state) 805 { 806 uint64_t ret; 807 808 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: dip 0x%p msiq_id 0x%x\n", 809 dip, msiq_id); 810 811 if ((ret = hvio_msiq_getvalid(DIP_TO_HANDLE(dip), 812 msiq_id, msiq_valid_state)) != H_EOK) { 813 DBG(DBG_LIB_MSIQ, dip, 814 "hvio_msiq_getvalid failed, ret 0x%lx\n", ret); 815 return (DDI_FAILURE); 816 } 817 818 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: msiq_valid_state 0x%x\n", 819 *msiq_valid_state); 820 821 return (DDI_SUCCESS); 822 } 823 824 /*ARGSUSED*/ 825 int 826 px_lib_msiq_setvalid(dev_info_t *dip, msiqid_t msiq_id, 827 pci_msiq_valid_state_t msiq_valid_state) 828 { 829 uint64_t ret; 830 831 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setvalid: dip 0x%p msiq_id 0x%x " 832 "msiq_valid_state 0x%x\n", dip, msiq_id, msiq_valid_state); 833 834 if ((ret = hvio_msiq_setvalid(DIP_TO_HANDLE(dip), 835 msiq_id, msiq_valid_state)) != H_EOK) { 836 DBG(DBG_LIB_MSIQ, dip, 837 "hvio_msiq_setvalid failed, ret 0x%lx\n", ret); 838 return (DDI_FAILURE); 839 } 840 841 return (DDI_SUCCESS); 842 } 843 844 /*ARGSUSED*/ 845 int 846 px_lib_msiq_getstate(dev_info_t *dip, msiqid_t msiq_id, 847 pci_msiq_state_t *msiq_state) 848 { 849 uint64_t ret; 850 851 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: dip 0x%p msiq_id 0x%x\n", 852 dip, msiq_id); 853 854 if ((ret = hvio_msiq_getstate(DIP_TO_HANDLE(dip), 855 msiq_id, msiq_state)) != H_EOK) { 856 DBG(DBG_LIB_MSIQ, dip, 857 "hvio_msiq_getstate failed, ret 0x%lx\n", ret); 858 return (DDI_FAILURE); 859 } 860 861 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: msiq_state 0x%x\n", 862 *msiq_state); 863 864 return (DDI_SUCCESS); 865 } 866 867 /*ARGSUSED*/ 868 int 869 px_lib_msiq_setstate(dev_info_t *dip, msiqid_t msiq_id, 870 pci_msiq_state_t msiq_state) 871 { 872 uint64_t ret; 873 874 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setstate: dip 0x%p msiq_id 0x%x " 875 "msiq_state 0x%x\n", dip, msiq_id, msiq_state); 876 877 if ((ret = hvio_msiq_setstate(DIP_TO_HANDLE(dip), 878 msiq_id, msiq_state)) != H_EOK) { 879 DBG(DBG_LIB_MSIQ, dip, 880 "hvio_msiq_setstate failed, ret 0x%lx\n", ret); 881 return (DDI_FAILURE); 882 } 883 884 return (DDI_SUCCESS); 885 } 886 887 /*ARGSUSED*/ 888 int 889 px_lib_msiq_gethead(dev_info_t *dip, msiqid_t msiq_id, 890 msiqhead_t *msiq_head_p) 891 { 892 uint64_t ret; 893 894 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: dip 0x%p msiq_id 0x%x\n", 895 dip, msiq_id); 896 897 if ((ret = hvio_msiq_gethead(DIP_TO_HANDLE(dip), 898 msiq_id, msiq_head_p)) != H_EOK) { 899 DBG(DBG_LIB_MSIQ, dip, 900 "hvio_msiq_gethead failed, ret 0x%lx\n", ret); 901 return (DDI_FAILURE); 902 } 903 904 *msiq_head_p = (*msiq_head_p / sizeof (msiq_rec_t)); 905 906 DBG(DBG_LIB_MSIQ, dip, "px_msiq_gethead: msiq_head 0x%x\n", 907 *msiq_head_p); 908 909 return (DDI_SUCCESS); 910 } 911 912 /*ARGSUSED*/ 913 int 914 px_lib_msiq_sethead(dev_info_t *dip, msiqid_t msiq_id, 915 msiqhead_t msiq_head) 916 { 917 uint64_t ret; 918 919 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_sethead: dip 0x%p msiq_id 0x%x " 920 "msiq_head 0x%x\n", dip, msiq_id, msiq_head); 921 922 if ((ret = hvio_msiq_sethead(DIP_TO_HANDLE(dip), 923 msiq_id, msiq_head * sizeof (msiq_rec_t))) != H_EOK) { 924 DBG(DBG_LIB_MSIQ, dip, 925 "hvio_msiq_sethead failed, ret 0x%lx\n", ret); 926 return (DDI_FAILURE); 927 } 928 929 return (DDI_SUCCESS); 930 } 931 932 /*ARGSUSED*/ 933 int 934 px_lib_msiq_gettail(dev_info_t *dip, msiqid_t msiq_id, 935 msiqtail_t *msiq_tail_p) 936 { 937 uint64_t ret; 938 939 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: dip 0x%p msiq_id 0x%x\n", 940 dip, msiq_id); 941 942 if ((ret = hvio_msiq_gettail(DIP_TO_HANDLE(dip), 943 msiq_id, msiq_tail_p)) != H_EOK) { 944 DBG(DBG_LIB_MSIQ, dip, 945 "hvio_msiq_gettail failed, ret 0x%lx\n", ret); 946 return (DDI_FAILURE); 947 } 948 949 *msiq_tail_p = (*msiq_tail_p / sizeof (msiq_rec_t)); 950 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: msiq_tail 0x%x\n", 951 *msiq_tail_p); 952 953 return (DDI_SUCCESS); 954 } 955 956 /*ARGSUSED*/ 957 void 958 px_lib_get_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p, 959 msiq_rec_t *msiq_rec_p) 960 { 961 msiq_rec_t *curr_msiq_rec_p = (msiq_rec_t *)msiq_head_p; 962 963 DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: dip 0x%p\n", dip); 964 965 if (!curr_msiq_rec_p->msiq_rec_type) { 966 /* Set msiq_rec_type to zero */ 967 msiq_rec_p->msiq_rec_type = 0; 968 969 return; 970 } 971 972 *msiq_rec_p = *curr_msiq_rec_p; 973 } 974 975 /*ARGSUSED*/ 976 void 977 px_lib_clr_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p) 978 { 979 msiq_rec_t *curr_msiq_rec_p = (msiq_rec_t *)msiq_head_p; 980 981 DBG(DBG_LIB_MSIQ, dip, "px_lib_clr_msiq_rec: dip 0x%p\n", dip); 982 983 /* Zero out msiq_rec_type field */ 984 curr_msiq_rec_p->msiq_rec_type = 0; 985 } 986 987 /* 988 * MSI Functions: 989 */ 990 991 /*ARGSUSED*/ 992 int 993 px_lib_msi_init(dev_info_t *dip) 994 { 995 DBG(DBG_LIB_MSI, dip, "px_lib_msi_init: dip 0x%p\n", dip); 996 997 /* Noop */ 998 return (DDI_SUCCESS); 999 } 1000 1001 /*ARGSUSED*/ 1002 int 1003 px_lib_msi_getmsiq(dev_info_t *dip, msinum_t msi_num, 1004 msiqid_t *msiq_id) 1005 { 1006 uint64_t ret; 1007 1008 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: dip 0x%p msi_num 0x%x\n", 1009 dip, msi_num); 1010 1011 if ((ret = hvio_msi_getmsiq(DIP_TO_HANDLE(dip), 1012 msi_num, msiq_id)) != H_EOK) { 1013 DBG(DBG_LIB_MSI, dip, 1014 "hvio_msi_getmsiq failed, ret 0x%lx\n", ret); 1015 return (DDI_FAILURE); 1016 } 1017 1018 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: msiq_id 0x%x\n", 1019 *msiq_id); 1020 1021 return (DDI_SUCCESS); 1022 } 1023 1024 /*ARGSUSED*/ 1025 int 1026 px_lib_msi_setmsiq(dev_info_t *dip, msinum_t msi_num, 1027 msiqid_t msiq_id, msi_type_t msitype) 1028 { 1029 uint64_t ret; 1030 1031 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setmsiq: dip 0x%p msi_num 0x%x " 1032 "msq_id 0x%x\n", dip, msi_num, msiq_id); 1033 1034 if ((ret = hvio_msi_setmsiq(DIP_TO_HANDLE(dip), 1035 msi_num, msiq_id, msitype)) != H_EOK) { 1036 DBG(DBG_LIB_MSI, dip, 1037 "hvio_msi_setmsiq failed, ret 0x%lx\n", ret); 1038 return (DDI_FAILURE); 1039 } 1040 1041 return (DDI_SUCCESS); 1042 } 1043 1044 /*ARGSUSED*/ 1045 int 1046 px_lib_msi_getvalid(dev_info_t *dip, msinum_t msi_num, 1047 pci_msi_valid_state_t *msi_valid_state) 1048 { 1049 uint64_t ret; 1050 1051 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: dip 0x%p msi_num 0x%x\n", 1052 dip, msi_num); 1053 1054 if ((ret = hvio_msi_getvalid(DIP_TO_HANDLE(dip), 1055 msi_num, msi_valid_state)) != H_EOK) { 1056 DBG(DBG_LIB_MSI, dip, 1057 "hvio_msi_getvalid failed, ret 0x%lx\n", ret); 1058 return (DDI_FAILURE); 1059 } 1060 1061 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: msiq_id 0x%x\n", 1062 *msi_valid_state); 1063 1064 return (DDI_SUCCESS); 1065 } 1066 1067 /*ARGSUSED*/ 1068 int 1069 px_lib_msi_setvalid(dev_info_t *dip, msinum_t msi_num, 1070 pci_msi_valid_state_t msi_valid_state) 1071 { 1072 uint64_t ret; 1073 1074 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setvalid: dip 0x%p msi_num 0x%x " 1075 "msi_valid_state 0x%x\n", dip, msi_num, msi_valid_state); 1076 1077 if ((ret = hvio_msi_setvalid(DIP_TO_HANDLE(dip), 1078 msi_num, msi_valid_state)) != H_EOK) { 1079 DBG(DBG_LIB_MSI, dip, 1080 "hvio_msi_setvalid failed, ret 0x%lx\n", ret); 1081 return (DDI_FAILURE); 1082 } 1083 1084 return (DDI_SUCCESS); 1085 } 1086 1087 /*ARGSUSED*/ 1088 int 1089 px_lib_msi_getstate(dev_info_t *dip, msinum_t msi_num, 1090 pci_msi_state_t *msi_state) 1091 { 1092 uint64_t ret; 1093 1094 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: dip 0x%p msi_num 0x%x\n", 1095 dip, msi_num); 1096 1097 if ((ret = hvio_msi_getstate(DIP_TO_HANDLE(dip), 1098 msi_num, msi_state)) != H_EOK) { 1099 DBG(DBG_LIB_MSI, dip, 1100 "hvio_msi_getstate failed, ret 0x%lx\n", ret); 1101 return (DDI_FAILURE); 1102 } 1103 1104 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: msi_state 0x%x\n", 1105 *msi_state); 1106 1107 return (DDI_SUCCESS); 1108 } 1109 1110 /*ARGSUSED*/ 1111 int 1112 px_lib_msi_setstate(dev_info_t *dip, msinum_t msi_num, 1113 pci_msi_state_t msi_state) 1114 { 1115 uint64_t ret; 1116 1117 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setstate: dip 0x%p msi_num 0x%x " 1118 "msi_state 0x%x\n", dip, msi_num, msi_state); 1119 1120 if ((ret = hvio_msi_setstate(DIP_TO_HANDLE(dip), 1121 msi_num, msi_state)) != H_EOK) { 1122 DBG(DBG_LIB_MSI, dip, 1123 "hvio_msi_setstate failed, ret 0x%lx\n", ret); 1124 return (DDI_FAILURE); 1125 } 1126 1127 return (DDI_SUCCESS); 1128 } 1129 1130 /* 1131 * MSG Functions: 1132 */ 1133 1134 /*ARGSUSED*/ 1135 int 1136 px_lib_msg_getmsiq(dev_info_t *dip, pcie_msg_type_t msg_type, 1137 msiqid_t *msiq_id) 1138 { 1139 uint64_t ret; 1140 1141 DBG(DBG_LIB_MSG, dip, "px_lib_msg_getmsiq: dip 0x%p msg_type 0x%x\n", 1142 dip, msg_type); 1143 1144 if ((ret = hvio_msg_getmsiq(DIP_TO_HANDLE(dip), 1145 msg_type, msiq_id)) != H_EOK) { 1146 DBG(DBG_LIB_MSG, dip, 1147 "hvio_msg_getmsiq failed, ret 0x%lx\n", ret); 1148 return (DDI_FAILURE); 1149 } 1150 1151 DBG(DBG_LIB_MSI, dip, "px_lib_msg_getmsiq: msiq_id 0x%x\n", 1152 *msiq_id); 1153 1154 return (DDI_SUCCESS); 1155 } 1156 1157 /*ARGSUSED*/ 1158 int 1159 px_lib_msg_setmsiq(dev_info_t *dip, pcie_msg_type_t msg_type, 1160 msiqid_t msiq_id) 1161 { 1162 uint64_t ret; 1163 1164 DBG(DBG_LIB_MSG, dip, "px_lib_msg_setmsiq: dip 0x%p msg_type 0x%x " 1165 "msq_id 0x%x\n", dip, msg_type, msiq_id); 1166 1167 if ((ret = hvio_msg_setmsiq(DIP_TO_HANDLE(dip), 1168 msg_type, msiq_id)) != H_EOK) { 1169 DBG(DBG_LIB_MSG, dip, 1170 "hvio_msg_setmsiq failed, ret 0x%lx\n", ret); 1171 return (DDI_FAILURE); 1172 } 1173 1174 return (DDI_SUCCESS); 1175 } 1176 1177 /*ARGSUSED*/ 1178 int 1179 px_lib_msg_getvalid(dev_info_t *dip, pcie_msg_type_t msg_type, 1180 pcie_msg_valid_state_t *msg_valid_state) 1181 { 1182 uint64_t ret; 1183 1184 DBG(DBG_LIB_MSG, dip, "px_lib_msg_getvalid: dip 0x%p msg_type 0x%x\n", 1185 dip, msg_type); 1186 1187 if ((ret = hvio_msg_getvalid(DIP_TO_HANDLE(dip), msg_type, 1188 msg_valid_state)) != H_EOK) { 1189 DBG(DBG_LIB_MSG, dip, 1190 "hvio_msg_getvalid failed, ret 0x%lx\n", ret); 1191 return (DDI_FAILURE); 1192 } 1193 1194 DBG(DBG_LIB_MSI, dip, "px_lib_msg_getvalid: msg_valid_state 0x%x\n", 1195 *msg_valid_state); 1196 1197 return (DDI_SUCCESS); 1198 } 1199 1200 /*ARGSUSED*/ 1201 int 1202 px_lib_msg_setvalid(dev_info_t *dip, pcie_msg_type_t msg_type, 1203 pcie_msg_valid_state_t msg_valid_state) 1204 { 1205 uint64_t ret; 1206 1207 DBG(DBG_LIB_MSG, dip, "px_lib_msg_setvalid: dip 0x%p msg_type 0x%x " 1208 "msg_valid_state 0x%x\n", dip, msg_type, msg_valid_state); 1209 1210 if ((ret = hvio_msg_setvalid(DIP_TO_HANDLE(dip), msg_type, 1211 msg_valid_state)) != H_EOK) { 1212 DBG(DBG_LIB_MSG, dip, 1213 "hvio_msg_setvalid failed, ret 0x%lx\n", ret); 1214 return (DDI_FAILURE); 1215 } 1216 1217 return (DDI_SUCCESS); 1218 } 1219 1220 /* 1221 * Suspend/Resume Functions: 1222 * Currently unsupported by hypervisor and all functions are noops. 1223 */ 1224 /*ARGSUSED*/ 1225 int 1226 px_lib_suspend(dev_info_t *dip) 1227 { 1228 DBG(DBG_ATTACH, dip, "px_lib_suspend: Not supported\n"); 1229 1230 /* Not supported */ 1231 return (DDI_FAILURE); 1232 } 1233 1234 /*ARGSUSED*/ 1235 void 1236 px_lib_resume(dev_info_t *dip) 1237 { 1238 DBG(DBG_ATTACH, dip, "px_lib_resume: Not supported\n"); 1239 1240 /* Noop */ 1241 } 1242 1243 /* 1244 * Misc Functions: 1245 * Currently unsupported by hypervisor and all functions are noops. 1246 */ 1247 /*ARGSUSED*/ 1248 static int 1249 px_lib_config_get(dev_info_t *dip, pci_device_t bdf, pci_config_offset_t off, 1250 uint8_t size, pci_cfg_data_t *data_p) 1251 { 1252 uint64_t ret; 1253 1254 DBG(DBG_LIB_CFG, dip, "px_lib_config_get: dip 0x%p, bdf 0x%llx " 1255 "off 0x%x size 0x%x\n", dip, bdf, off, size); 1256 1257 if ((ret = hvio_config_get(DIP_TO_HANDLE(dip), bdf, off, 1258 size, data_p)) != H_EOK) { 1259 DBG(DBG_LIB_CFG, dip, 1260 "hvio_config_get failed, ret 0x%lx\n", ret); 1261 return (DDI_FAILURE); 1262 } 1263 DBG(DBG_LIB_CFG, dip, "px_config_get: data 0x%x\n", data_p->dw); 1264 1265 return (DDI_SUCCESS); 1266 } 1267 1268 /*ARGSUSED*/ 1269 static int 1270 px_lib_config_put(dev_info_t *dip, pci_device_t bdf, pci_config_offset_t off, 1271 uint8_t size, pci_cfg_data_t data) 1272 { 1273 uint64_t ret; 1274 1275 DBG(DBG_LIB_CFG, dip, "px_lib_config_put: dip 0x%p, bdf 0x%llx " 1276 "off 0x%x size 0x%x data 0x%llx\n", dip, bdf, off, size, data.qw); 1277 1278 if ((ret = hvio_config_put(DIP_TO_HANDLE(dip), bdf, off, 1279 size, data)) != H_EOK) { 1280 DBG(DBG_LIB_CFG, dip, 1281 "hvio_config_put failed, ret 0x%lx\n", ret); 1282 return (DDI_FAILURE); 1283 } 1284 1285 return (DDI_SUCCESS); 1286 } 1287 1288 static uint32_t 1289 px_pci_config_get(ddi_acc_impl_t *handle, uint32_t *addr, int size) 1290 { 1291 px_config_acc_pvt_t *px_pvt = (px_config_acc_pvt_t *) 1292 handle->ahi_common.ah_bus_private; 1293 pcie_bus_t *busp = NULL; 1294 dev_info_t *cdip = NULL; 1295 uint32_t pci_dev_addr = px_pvt->raddr; 1296 uint32_t vaddr = px_pvt->vaddr; 1297 uint16_t off = (uint16_t)(uintptr_t)(addr - vaddr) & 0xfff; 1298 uint64_t rdata = 0; 1299 1300 if (px_lib_config_get(px_pvt->dip, pci_dev_addr, off, 1301 size, (pci_cfg_data_t *)&rdata) != DDI_SUCCESS) 1302 /* XXX update error kstats */ 1303 return (0xffffffff); 1304 1305 if (cdip = pcie_find_dip_by_bdf(px_pvt->dip, pci_dev_addr >> 8)) 1306 busp = PCIE_DIP2BUS(cdip); 1307 /* 1308 * This can be called early, before busp or busp->bus_dom has 1309 * been initialized, so check both before invoking 1310 * PCIE_IS_ASSIGNED. 1311 */ 1312 if (busp && PCIE_BUS2DOM(busp) && PCIE_IS_ASSIGNED(busp)) { 1313 if (off == PCI_CONF_VENID && size == 2) 1314 rdata = busp->bus_dev_ven_id & 0xffff; 1315 else if (off == PCI_CONF_DEVID && size == 2) 1316 rdata = busp->bus_dev_ven_id >> 16; 1317 else if (off == PCI_CONF_VENID && size == 4) 1318 rdata = busp->bus_dev_ven_id; 1319 } 1320 return ((uint32_t)rdata); 1321 } 1322 1323 static void 1324 px_pci_config_put(ddi_acc_impl_t *handle, uint32_t *addr, 1325 int size, pci_cfg_data_t wdata) 1326 { 1327 px_config_acc_pvt_t *px_pvt = (px_config_acc_pvt_t *) 1328 handle->ahi_common.ah_bus_private; 1329 uint32_t pci_dev_addr = px_pvt->raddr; 1330 uint32_t vaddr = px_pvt->vaddr; 1331 uint16_t off = (uint16_t)(uintptr_t)(addr - vaddr) & 0xfff; 1332 1333 if (px_lib_config_put(px_pvt->dip, pci_dev_addr, off, 1334 size, wdata) != DDI_SUCCESS) { 1335 /*EMPTY*/ 1336 /* XXX update error kstats */ 1337 } 1338 } 1339 1340 static uint8_t 1341 px_pci_config_get8(ddi_acc_impl_t *handle, uint8_t *addr) 1342 { 1343 return ((uint8_t)px_pci_config_get(handle, (uint32_t *)addr, 1)); 1344 } 1345 1346 static uint16_t 1347 px_pci_config_get16(ddi_acc_impl_t *handle, uint16_t *addr) 1348 { 1349 return ((uint16_t)px_pci_config_get(handle, (uint32_t *)addr, 2)); 1350 } 1351 1352 static uint32_t 1353 px_pci_config_get32(ddi_acc_impl_t *handle, uint32_t *addr) 1354 { 1355 return ((uint32_t)px_pci_config_get(handle, (uint32_t *)addr, 4)); 1356 } 1357 1358 static uint64_t 1359 px_pci_config_get64(ddi_acc_impl_t *handle, uint64_t *addr) 1360 { 1361 uint32_t rdatah, rdatal; 1362 1363 rdatal = (uint32_t)px_pci_config_get(handle, (uint32_t *)addr, 4); 1364 rdatah = (uint32_t)px_pci_config_get(handle, 1365 (uint32_t *)((char *)addr+4), 4); 1366 return (((uint64_t)rdatah << 32) | rdatal); 1367 } 1368 1369 static void 1370 px_pci_config_put8(ddi_acc_impl_t *handle, uint8_t *addr, uint8_t data) 1371 { 1372 pci_cfg_data_t wdata = { 0 }; 1373 1374 wdata.qw = (uint8_t)data; 1375 px_pci_config_put(handle, (uint32_t *)addr, 1, wdata); 1376 } 1377 1378 static void 1379 px_pci_config_put16(ddi_acc_impl_t *handle, uint16_t *addr, uint16_t data) 1380 { 1381 pci_cfg_data_t wdata = { 0 }; 1382 1383 wdata.qw = (uint16_t)data; 1384 px_pci_config_put(handle, (uint32_t *)addr, 2, wdata); 1385 } 1386 1387 static void 1388 px_pci_config_put32(ddi_acc_impl_t *handle, uint32_t *addr, uint32_t data) 1389 { 1390 pci_cfg_data_t wdata = { 0 }; 1391 1392 wdata.qw = (uint32_t)data; 1393 px_pci_config_put(handle, (uint32_t *)addr, 4, wdata); 1394 } 1395 1396 static void 1397 px_pci_config_put64(ddi_acc_impl_t *handle, uint64_t *addr, uint64_t data) 1398 { 1399 pci_cfg_data_t wdata = { 0 }; 1400 1401 wdata.qw = (uint32_t)(data & 0xffffffff); 1402 px_pci_config_put(handle, (uint32_t *)addr, 4, wdata); 1403 wdata.qw = (uint32_t)((data >> 32) & 0xffffffff); 1404 px_pci_config_put(handle, (uint32_t *)((char *)addr+4), 4, wdata); 1405 } 1406 1407 static void 1408 px_pci_config_rep_get8(ddi_acc_impl_t *handle, uint8_t *host_addr, 1409 uint8_t *dev_addr, size_t repcount, uint_t flags) 1410 { 1411 if (flags == DDI_DEV_AUTOINCR) 1412 for (; repcount; repcount--) 1413 *host_addr++ = px_pci_config_get8(handle, dev_addr++); 1414 else 1415 for (; repcount; repcount--) 1416 *host_addr++ = px_pci_config_get8(handle, dev_addr); 1417 } 1418 1419 /* 1420 * Function to rep read 16 bit data off the PCI configuration space behind 1421 * the 21554's host interface. 1422 */ 1423 static void 1424 px_pci_config_rep_get16(ddi_acc_impl_t *handle, uint16_t *host_addr, 1425 uint16_t *dev_addr, size_t repcount, uint_t flags) 1426 { 1427 if (flags == DDI_DEV_AUTOINCR) 1428 for (; repcount; repcount--) 1429 *host_addr++ = px_pci_config_get16(handle, dev_addr++); 1430 else 1431 for (; repcount; repcount--) 1432 *host_addr++ = px_pci_config_get16(handle, dev_addr); 1433 } 1434 1435 /* 1436 * Function to rep read 32 bit data off the PCI configuration space behind 1437 * the 21554's host interface. 1438 */ 1439 static void 1440 px_pci_config_rep_get32(ddi_acc_impl_t *handle, uint32_t *host_addr, 1441 uint32_t *dev_addr, size_t repcount, uint_t flags) 1442 { 1443 if (flags == DDI_DEV_AUTOINCR) 1444 for (; repcount; repcount--) 1445 *host_addr++ = px_pci_config_get32(handle, dev_addr++); 1446 else 1447 for (; repcount; repcount--) 1448 *host_addr++ = px_pci_config_get32(handle, dev_addr); 1449 } 1450 1451 /* 1452 * Function to rep read 64 bit data off the PCI configuration space behind 1453 * the 21554's host interface. 1454 */ 1455 static void 1456 px_pci_config_rep_get64(ddi_acc_impl_t *handle, uint64_t *host_addr, 1457 uint64_t *dev_addr, size_t repcount, uint_t flags) 1458 { 1459 if (flags == DDI_DEV_AUTOINCR) 1460 for (; repcount; repcount--) 1461 *host_addr++ = px_pci_config_get64(handle, dev_addr++); 1462 else 1463 for (; repcount; repcount--) 1464 *host_addr++ = px_pci_config_get64(handle, dev_addr); 1465 } 1466 1467 /* 1468 * Function to rep write 8 bit data into the PCI configuration space behind 1469 * the 21554's host interface. 1470 */ 1471 static void 1472 px_pci_config_rep_put8(ddi_acc_impl_t *handle, uint8_t *host_addr, 1473 uint8_t *dev_addr, size_t repcount, uint_t flags) 1474 { 1475 if (flags == DDI_DEV_AUTOINCR) 1476 for (; repcount; repcount--) 1477 px_pci_config_put8(handle, dev_addr++, *host_addr++); 1478 else 1479 for (; repcount; repcount--) 1480 px_pci_config_put8(handle, dev_addr, *host_addr++); 1481 } 1482 1483 /* 1484 * Function to rep write 16 bit data into the PCI configuration space behind 1485 * the 21554's host interface. 1486 */ 1487 static void 1488 px_pci_config_rep_put16(ddi_acc_impl_t *handle, uint16_t *host_addr, 1489 uint16_t *dev_addr, size_t repcount, uint_t flags) 1490 { 1491 if (flags == DDI_DEV_AUTOINCR) 1492 for (; repcount; repcount--) 1493 px_pci_config_put16(handle, dev_addr++, *host_addr++); 1494 else 1495 for (; repcount; repcount--) 1496 px_pci_config_put16(handle, dev_addr, *host_addr++); 1497 } 1498 1499 /* 1500 * Function to rep write 32 bit data into the PCI configuration space behind 1501 * the 21554's host interface. 1502 */ 1503 static void 1504 px_pci_config_rep_put32(ddi_acc_impl_t *handle, uint32_t *host_addr, 1505 uint32_t *dev_addr, size_t repcount, uint_t flags) 1506 { 1507 if (flags == DDI_DEV_AUTOINCR) 1508 for (; repcount; repcount--) 1509 px_pci_config_put32(handle, dev_addr++, *host_addr++); 1510 else 1511 for (; repcount; repcount--) 1512 px_pci_config_put32(handle, dev_addr, *host_addr++); 1513 } 1514 1515 /* 1516 * Function to rep write 64 bit data into the PCI configuration space behind 1517 * the 21554's host interface. 1518 */ 1519 static void 1520 px_pci_config_rep_put64(ddi_acc_impl_t *handle, uint64_t *host_addr, 1521 uint64_t *dev_addr, size_t repcount, uint_t flags) 1522 { 1523 if (flags == DDI_DEV_AUTOINCR) 1524 for (; repcount; repcount--) 1525 px_pci_config_put64(handle, dev_addr++, *host_addr++); 1526 else 1527 for (; repcount; repcount--) 1528 px_pci_config_put64(handle, dev_addr, *host_addr++); 1529 } 1530 1531 /* 1532 * Provide a private access handle to route config access calls to Hypervisor. 1533 * Beware: Do all error checking for config space accesses before calling 1534 * this function. ie. do error checking from the calling function. 1535 * Due to a lack of meaningful error code in DDI, the gauranteed return of 1536 * DDI_SUCCESS from here makes the code organization readable/easier from 1537 * the generic code. 1538 */ 1539 /*ARGSUSED*/ 1540 int 1541 px_lib_map_vconfig(dev_info_t *dip, 1542 ddi_map_req_t *mp, pci_config_offset_t off, 1543 pci_regspec_t *rp, caddr_t *addrp) 1544 { 1545 int fmcap; 1546 ndi_err_t *errp; 1547 on_trap_data_t *otp; 1548 ddi_acc_hdl_t *hp; 1549 ddi_acc_impl_t *ap; 1550 uchar_t busnum; /* bus number */ 1551 uchar_t devnum; /* device number */ 1552 uchar_t funcnum; /* function number */ 1553 px_config_acc_pvt_t *px_pvt; 1554 1555 hp = (ddi_acc_hdl_t *)mp->map_handlep; 1556 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1557 1558 /* Check for mapping teardown operation */ 1559 if ((mp->map_op == DDI_MO_UNMAP) || 1560 (mp->map_op == DDI_MO_UNLOCK)) { 1561 /* free up memory allocated for the private access handle. */ 1562 px_pvt = (px_config_acc_pvt_t *)hp->ah_bus_private; 1563 kmem_free((void *)px_pvt, sizeof (px_config_acc_pvt_t)); 1564 1565 /* unmap operation of PCI IO/config space. */ 1566 return (DDI_SUCCESS); 1567 } 1568 1569 fmcap = ddi_fm_capable(dip); 1570 if (DDI_FM_ACC_ERR_CAP(fmcap)) { 1571 errp = ((ddi_acc_impl_t *)hp)->ahi_err; 1572 otp = (on_trap_data_t *)errp->err_ontrap; 1573 otp->ot_handle = (void *)(hp); 1574 otp->ot_prot = OT_DATA_ACCESS; 1575 errp->err_status = DDI_FM_OK; 1576 errp->err_expected = DDI_FM_ERR_UNEXPECTED; 1577 errp->err_cf = px_err_cfg_hdl_check; 1578 } 1579 1580 ap->ahi_get8 = px_pci_config_get8; 1581 ap->ahi_get16 = px_pci_config_get16; 1582 ap->ahi_get32 = px_pci_config_get32; 1583 ap->ahi_get64 = px_pci_config_get64; 1584 ap->ahi_put8 = px_pci_config_put8; 1585 ap->ahi_put16 = px_pci_config_put16; 1586 ap->ahi_put32 = px_pci_config_put32; 1587 ap->ahi_put64 = px_pci_config_put64; 1588 ap->ahi_rep_get8 = px_pci_config_rep_get8; 1589 ap->ahi_rep_get16 = px_pci_config_rep_get16; 1590 ap->ahi_rep_get32 = px_pci_config_rep_get32; 1591 ap->ahi_rep_get64 = px_pci_config_rep_get64; 1592 ap->ahi_rep_put8 = px_pci_config_rep_put8; 1593 ap->ahi_rep_put16 = px_pci_config_rep_put16; 1594 ap->ahi_rep_put32 = px_pci_config_rep_put32; 1595 ap->ahi_rep_put64 = px_pci_config_rep_put64; 1596 1597 /* Initialize to default check/notify functions */ 1598 ap->ahi_fault = 0; 1599 ap->ahi_fault_check = i_ddi_acc_fault_check; 1600 ap->ahi_fault_notify = i_ddi_acc_fault_notify; 1601 1602 /* allocate memory for our private handle */ 1603 px_pvt = (px_config_acc_pvt_t *) 1604 kmem_zalloc(sizeof (px_config_acc_pvt_t), KM_SLEEP); 1605 hp->ah_bus_private = (void *)px_pvt; 1606 1607 busnum = PCI_REG_BUS_G(rp->pci_phys_hi); 1608 devnum = PCI_REG_DEV_G(rp->pci_phys_hi); 1609 funcnum = PCI_REG_FUNC_G(rp->pci_phys_hi); 1610 1611 /* set up private data for use during IO routines */ 1612 1613 /* addr needed by the HV APIs */ 1614 px_pvt->raddr = busnum << 16 | devnum << 11 | funcnum << 8; 1615 /* 1616 * Address that specifies the actual offset into the 256MB 1617 * memory mapped configuration space, 4K per device. 1618 * First 12bits form the offset into 4K config space. 1619 * This address is only used during the IO routines to calculate 1620 * the offset at which the transaction must be performed. 1621 * Drivers bypassing DDI functions to access PCI config space will 1622 * panic the system since the following is a bogus virtual address. 1623 */ 1624 px_pvt->vaddr = busnum << 20 | devnum << 15 | funcnum << 12 | off; 1625 px_pvt->dip = dip; 1626 1627 DBG(DBG_LIB_CFG, dip, "px_config_setup: raddr 0x%x, vaddr 0x%x\n", 1628 px_pvt->raddr, px_pvt->vaddr); 1629 *addrp = (caddr_t)(uintptr_t)px_pvt->vaddr; 1630 return (DDI_SUCCESS); 1631 } 1632 1633 /*ARGSUSED*/ 1634 void 1635 px_lib_map_attr_check(ddi_map_req_t *mp) 1636 { 1637 } 1638 1639 /* 1640 * px_lib_log_safeacc_err: 1641 * Imitate a cpu/mem trap call when a peek/poke fails. 1642 * This will initiate something similar to px_fm_callback. 1643 */ 1644 static void 1645 px_lib_log_safeacc_err(px_t *px_p, ddi_acc_handle_t handle, int fme_flag, 1646 r_addr_t addr) 1647 { 1648 uint32_t addr_high, addr_low; 1649 pcie_req_id_t bdf = PCIE_INVALID_BDF; 1650 pci_ranges_t *ranges_p; 1651 int range_len, i; 1652 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle; 1653 ddi_fm_error_t derr; 1654 1655 if (px_fm_enter(px_p) != DDI_SUCCESS) 1656 return; 1657 1658 derr.fme_status = DDI_FM_NONFATAL; 1659 derr.fme_version = DDI_FME_VERSION; 1660 derr.fme_flag = fme_flag; 1661 derr.fme_ena = fm_ena_generate(0, FM_ENA_FMT1); 1662 derr.fme_acc_handle = handle; 1663 if (hp) 1664 hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED; 1665 1666 addr_high = (uint32_t)(addr >> 32); 1667 addr_low = (uint32_t)addr; 1668 1669 /* 1670 * Make sure this failed load came from this PCIe port. Check by 1671 * matching the upper 32 bits of the address with the ranges property. 1672 */ 1673 range_len = px_p->px_ranges_length / sizeof (pci_ranges_t); 1674 i = 0; 1675 for (ranges_p = px_p->px_ranges_p; i < range_len; i++, ranges_p++) { 1676 if (ranges_p->parent_high == addr_high) { 1677 switch (ranges_p->child_high & PCI_ADDR_MASK) { 1678 case PCI_ADDR_CONFIG: 1679 bdf = (pcie_req_id_t)(addr_low >> 12); 1680 break; 1681 default: 1682 bdf = PCIE_INVALID_BDF; 1683 break; 1684 } 1685 break; 1686 } 1687 } 1688 1689 (void) px_rp_en_q(px_p, bdf, addr, 0); 1690 (void) px_scan_fabric(px_p, px_p->px_dip, &derr); 1691 px_fm_exit(px_p); 1692 } 1693 1694 1695 #ifdef DEBUG 1696 int px_peekfault_cnt = 0; 1697 int px_pokefault_cnt = 0; 1698 #endif /* DEBUG */ 1699 1700 /* 1701 * Do a safe write to a device. 1702 * 1703 * When this function is given a handle (cautious access), all errors are 1704 * suppressed. 1705 * 1706 * When this function is not given a handle (poke), only Unsupported Request 1707 * and Completer Abort errors are suppressed. 1708 * 1709 * In all cases, all errors are returned in the function return status. 1710 */ 1711 1712 int 1713 px_lib_ctlops_poke(dev_info_t *dip, dev_info_t *rdip, 1714 peekpoke_ctlops_t *in_args) 1715 { 1716 px_t *px_p = DIP_TO_STATE(dip); 1717 px_pec_t *pec_p = px_p->px_pec_p; 1718 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle; 1719 1720 size_t repcount = in_args->repcount; 1721 size_t size = in_args->size; 1722 uintptr_t dev_addr = in_args->dev_addr; 1723 uintptr_t host_addr = in_args->host_addr; 1724 1725 int err = DDI_SUCCESS; 1726 uint64_t hvio_poke_status; 1727 uint32_t wrt_stat; 1728 1729 r_addr_t ra; 1730 uint64_t pokeval; 1731 pcie_req_id_t bdf; 1732 1733 ra = (r_addr_t)va_to_pa((void *)dev_addr); 1734 for (; repcount; repcount--) { 1735 1736 switch (size) { 1737 case sizeof (uint8_t): 1738 pokeval = *(uint8_t *)host_addr; 1739 break; 1740 case sizeof (uint16_t): 1741 pokeval = *(uint16_t *)host_addr; 1742 break; 1743 case sizeof (uint32_t): 1744 pokeval = *(uint32_t *)host_addr; 1745 break; 1746 case sizeof (uint64_t): 1747 pokeval = *(uint64_t *)host_addr; 1748 break; 1749 default: 1750 DBG(DBG_MAP, px_p->px_dip, 1751 "poke: invalid size %d passed\n", size); 1752 err = DDI_FAILURE; 1753 goto done; 1754 } 1755 1756 /* 1757 * Grab pokefault mutex since hypervisor does not guarantee 1758 * poke serialization. 1759 */ 1760 if (hp) { 1761 i_ndi_busop_access_enter(hp->ahi_common.ah_dip, 1762 (ddi_acc_handle_t)hp); 1763 pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED; 1764 } else { 1765 mutex_enter(&pec_p->pec_pokefault_mutex); 1766 pec_p->pec_safeacc_type = DDI_FM_ERR_POKE; 1767 } 1768 1769 if (pcie_get_bdf_from_dip(rdip, &bdf) != DDI_SUCCESS) { 1770 err = DDI_FAILURE; 1771 goto done; 1772 } 1773 1774 hvio_poke_status = hvio_poke(px_p->px_dev_hdl, ra, size, 1775 pokeval, bdf << 8, &wrt_stat); 1776 1777 if ((hvio_poke_status != H_EOK) || (wrt_stat != H_EOK)) { 1778 err = DDI_FAILURE; 1779 #ifdef DEBUG 1780 px_pokefault_cnt++; 1781 #endif 1782 /* 1783 * For CAUTIOUS and POKE access, notify FMA to 1784 * cleanup. Imitate a cpu/mem trap call like in sun4u. 1785 */ 1786 px_lib_log_safeacc_err(px_p, (ddi_acc_handle_t)hp, 1787 (hp ? DDI_FM_ERR_EXPECTED : 1788 DDI_FM_ERR_POKE), ra); 1789 1790 pec_p->pec_ontrap_data = NULL; 1791 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1792 if (hp) { 1793 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, 1794 (ddi_acc_handle_t)hp); 1795 } else { 1796 mutex_exit(&pec_p->pec_pokefault_mutex); 1797 } 1798 goto done; 1799 } 1800 1801 pec_p->pec_ontrap_data = NULL; 1802 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1803 if (hp) { 1804 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, 1805 (ddi_acc_handle_t)hp); 1806 } else { 1807 mutex_exit(&pec_p->pec_pokefault_mutex); 1808 } 1809 1810 host_addr += size; 1811 1812 if (in_args->flags == DDI_DEV_AUTOINCR) { 1813 dev_addr += size; 1814 ra = (r_addr_t)va_to_pa((void *)dev_addr); 1815 } 1816 } 1817 1818 done: 1819 return (err); 1820 } 1821 1822 1823 /*ARGSUSED*/ 1824 int 1825 px_lib_ctlops_peek(dev_info_t *dip, dev_info_t *rdip, 1826 peekpoke_ctlops_t *in_args, void *result) 1827 { 1828 px_t *px_p = DIP_TO_STATE(dip); 1829 px_pec_t *pec_p = px_p->px_pec_p; 1830 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle; 1831 1832 size_t repcount = in_args->repcount; 1833 uintptr_t dev_addr = in_args->dev_addr; 1834 uintptr_t host_addr = in_args->host_addr; 1835 1836 r_addr_t ra; 1837 uint32_t read_status; 1838 uint64_t hvio_peek_status; 1839 uint64_t peekval; 1840 int err = DDI_SUCCESS; 1841 1842 result = (void *)in_args->host_addr; 1843 1844 ra = (r_addr_t)va_to_pa((void *)dev_addr); 1845 for (; repcount; repcount--) { 1846 1847 /* Lock pokefault mutex so read doesn't mask a poke fault. */ 1848 if (hp) { 1849 i_ndi_busop_access_enter(hp->ahi_common.ah_dip, 1850 (ddi_acc_handle_t)hp); 1851 pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED; 1852 } else { 1853 mutex_enter(&pec_p->pec_pokefault_mutex); 1854 pec_p->pec_safeacc_type = DDI_FM_ERR_PEEK; 1855 } 1856 1857 hvio_peek_status = hvio_peek(px_p->px_dev_hdl, ra, 1858 in_args->size, &read_status, &peekval); 1859 1860 if ((hvio_peek_status != H_EOK) || (read_status != H_EOK)) { 1861 err = DDI_FAILURE; 1862 1863 /* 1864 * For CAUTIOUS and PEEK access, notify FMA to 1865 * cleanup. Imitate a cpu/mem trap call like in sun4u. 1866 */ 1867 px_lib_log_safeacc_err(px_p, (ddi_acc_handle_t)hp, 1868 (hp ? DDI_FM_ERR_EXPECTED : 1869 DDI_FM_ERR_PEEK), ra); 1870 1871 /* Stuff FFs in host addr if peek. */ 1872 if (hp == NULL) { 1873 int i; 1874 uint8_t *ff_addr = (uint8_t *)host_addr; 1875 for (i = 0; i < in_args->size; i++) 1876 *ff_addr++ = 0xff; 1877 } 1878 #ifdef DEBUG 1879 px_peekfault_cnt++; 1880 #endif 1881 pec_p->pec_ontrap_data = NULL; 1882 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1883 if (hp) { 1884 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, 1885 (ddi_acc_handle_t)hp); 1886 } else { 1887 mutex_exit(&pec_p->pec_pokefault_mutex); 1888 } 1889 goto done; 1890 1891 } 1892 pec_p->pec_ontrap_data = NULL; 1893 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1894 if (hp) { 1895 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, 1896 (ddi_acc_handle_t)hp); 1897 } else { 1898 mutex_exit(&pec_p->pec_pokefault_mutex); 1899 } 1900 1901 switch (in_args->size) { 1902 case sizeof (uint8_t): 1903 *(uint8_t *)host_addr = (uint8_t)peekval; 1904 break; 1905 case sizeof (uint16_t): 1906 *(uint16_t *)host_addr = (uint16_t)peekval; 1907 break; 1908 case sizeof (uint32_t): 1909 *(uint32_t *)host_addr = (uint32_t)peekval; 1910 break; 1911 case sizeof (uint64_t): 1912 *(uint64_t *)host_addr = (uint64_t)peekval; 1913 break; 1914 default: 1915 DBG(DBG_MAP, px_p->px_dip, 1916 "peek: invalid size %d passed\n", 1917 in_args->size); 1918 err = DDI_FAILURE; 1919 goto done; 1920 } 1921 1922 host_addr += in_args->size; 1923 1924 if (in_args->flags == DDI_DEV_AUTOINCR) { 1925 dev_addr += in_args->size; 1926 ra = (r_addr_t)va_to_pa((void *)dev_addr); 1927 } 1928 } 1929 done: 1930 return (err); 1931 } 1932 1933 1934 /* add interrupt vector */ 1935 int 1936 px_err_add_intr(px_fault_t *px_fault_p) 1937 { 1938 px_t *px_p = DIP_TO_STATE(px_fault_p->px_fh_dip); 1939 1940 DBG(DBG_LIB_INT, px_p->px_dip, 1941 "px_err_add_intr: calling add_ivintr"); 1942 1943 VERIFY(add_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL, 1944 (intrfunc)px_fault_p->px_err_func, (caddr_t)px_fault_p, NULL, 1945 (caddr_t)&px_fault_p->px_intr_payload[0]) == 0); 1946 1947 DBG(DBG_LIB_INT, px_p->px_dip, 1948 "px_err_add_intr: ib_intr_enable "); 1949 1950 px_ib_intr_enable(px_p, intr_dist_cpuid(), px_fault_p->px_intr_ino); 1951 1952 return (DDI_SUCCESS); 1953 } 1954 1955 /* remove interrupt vector */ 1956 void 1957 px_err_rem_intr(px_fault_t *px_fault_p) 1958 { 1959 px_t *px_p = DIP_TO_STATE(px_fault_p->px_fh_dip); 1960 1961 px_ib_intr_disable(px_p->px_ib_p, px_fault_p->px_intr_ino, 1962 IB_INTR_WAIT); 1963 1964 VERIFY(rem_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL) == 0); 1965 } 1966 1967 void 1968 px_cb_intr_redist(void *arg) 1969 { 1970 px_t *px_p = (px_t *)arg; 1971 px_ib_intr_dist_en(px_p->px_dip, intr_dist_cpuid(), 1972 px_p->px_inos[PX_INTR_XBC], B_FALSE); 1973 } 1974 1975 int 1976 px_cb_add_intr(px_fault_t *f_p) 1977 { 1978 px_t *px_p = DIP_TO_STATE(f_p->px_fh_dip); 1979 1980 DBG(DBG_LIB_INT, px_p->px_dip, 1981 "px_err_add_intr: calling add_ivintr"); 1982 1983 VERIFY(add_ivintr(f_p->px_fh_sysino, PX_ERR_PIL, 1984 (intrfunc)f_p->px_err_func, (caddr_t)f_p, NULL, 1985 (caddr_t)&f_p->px_intr_payload[0]) == 0); 1986 1987 intr_dist_add(px_cb_intr_redist, px_p); 1988 1989 DBG(DBG_LIB_INT, px_p->px_dip, 1990 "px_err_add_intr: ib_intr_enable "); 1991 1992 px_ib_intr_enable(px_p, intr_dist_cpuid(), f_p->px_intr_ino); 1993 1994 return (DDI_SUCCESS); 1995 } 1996 1997 void 1998 px_cb_rem_intr(px_fault_t *f_p) 1999 { 2000 intr_dist_rem(px_cb_intr_redist, DIP_TO_STATE(f_p->px_fh_dip)); 2001 px_err_rem_intr(f_p); 2002 } 2003 2004 #ifdef FMA 2005 void 2006 px_fill_rc_status(px_fault_t *px_fault_p, pciex_rc_error_regs_t *rc_status) 2007 { 2008 px_pec_err_t *err_pkt; 2009 2010 err_pkt = (px_pec_err_t *)px_fault_p->px_intr_payload; 2011 2012 /* initialise all the structure members */ 2013 rc_status->status_valid = 0; 2014 2015 if (err_pkt->pec_descr.P) { 2016 /* PCI Status Register */ 2017 rc_status->pci_err_status = err_pkt->pci_err_status; 2018 rc_status->status_valid |= PCI_ERR_STATUS_VALID; 2019 } 2020 2021 if (err_pkt->pec_descr.E) { 2022 /* PCIe Status Register */ 2023 rc_status->pcie_err_status = err_pkt->pcie_err_status; 2024 rc_status->status_valid |= PCIE_ERR_STATUS_VALID; 2025 } 2026 2027 if (err_pkt->pec_descr.U) { 2028 rc_status->ue_status = err_pkt->ue_reg_status; 2029 rc_status->status_valid |= UE_STATUS_VALID; 2030 } 2031 2032 if (err_pkt->pec_descr.H) { 2033 rc_status->ue_hdr1 = err_pkt->hdr[0]; 2034 rc_status->status_valid |= UE_HDR1_VALID; 2035 } 2036 2037 if (err_pkt->pec_descr.I) { 2038 rc_status->ue_hdr2 = err_pkt->hdr[1]; 2039 rc_status->status_valid |= UE_HDR2_VALID; 2040 } 2041 2042 /* ue_fst_err_ptr - not available for sun4v?? */ 2043 2044 2045 if (err_pkt->pec_descr.S) { 2046 rc_status->source_id = err_pkt->err_src_reg; 2047 rc_status->status_valid |= SOURCE_ID_VALID; 2048 } 2049 2050 if (err_pkt->pec_descr.R) { 2051 rc_status->root_err_status = err_pkt->root_err_status; 2052 rc_status->status_valid |= CE_STATUS_VALID; 2053 } 2054 } 2055 #endif 2056 2057 /*ARGSUSED*/ 2058 int 2059 px_lib_pmctl(int cmd, px_t *px_p) 2060 { 2061 return (DDI_FAILURE); 2062 } 2063 2064 /*ARGSUSED*/ 2065 uint_t 2066 px_pmeq_intr(caddr_t arg) 2067 { 2068 return (DDI_INTR_CLAIMED); 2069 } 2070 2071 /* 2072 * fetch the config space base addr of the root complex 2073 * note this depends on px structure being initialized 2074 */ 2075 uint64_t 2076 px_lib_get_cfgacc_base(dev_info_t *dip) 2077 { 2078 int instance = DIP_TO_INST(dip); 2079 px_t *px_p = INST_TO_STATE(instance); 2080 2081 return (px_p->px_dev_hdl); 2082 } 2083 2084 void 2085 px_panic_domain(px_t *px_p, pcie_req_id_t bdf) 2086 { 2087 uint64_t ret; 2088 dev_info_t *dip = px_p->px_dip; 2089 2090 DBG(DBG_ERR_INTR, dip, "px_panic_domain: handle 0x%lx, ino %d, " 2091 "bdf<<8 0x%lx\n", 2092 (uint64_t)DIP_TO_HANDLE(dip), px_p->px_cb_fault.px_intr_ino, 2093 (pci_device_t)bdf << 8); 2094 if ((ret = pci_error_send(DIP_TO_HANDLE(dip), 2095 px_p->px_cb_fault.px_intr_ino, (pci_device_t)bdf << 8)) != H_EOK) { 2096 DBG(DBG_ERR_INTR, dip, "pci_error_send failed, ret 0x%lx\n", 2097 ret); 2098 } else 2099 DBG(DBG_ERR_INTR, dip, "pci_error_send worked\n"); 2100 } 2101 2102 /*ARGSUSED*/ 2103 int 2104 px_lib_hotplug_init(dev_info_t *dip, void *arg) 2105 { 2106 return (DDI_ENOTSUP); 2107 } 2108 2109 /*ARGSUSED*/ 2110 void 2111 px_lib_hotplug_uninit(dev_info_t *dip) 2112 { 2113 } 2114 2115 /*ARGSUSED*/ 2116 void 2117 px_hp_intr_redist(px_t *px_p) 2118 { 2119 } 2120 2121 /* Dummy cpr add callback */ 2122 /*ARGSUSED*/ 2123 void 2124 px_cpr_add_callb(px_t *px_p) 2125 { 2126 } 2127 2128 /* Dummy cpr rem callback */ 2129 /*ARGSUSED*/ 2130 void 2131 px_cpr_rem_callb(px_t *px_p) 2132 { 2133 } 2134 2135 /*ARGSUSED*/ 2136 boolean_t 2137 px_lib_is_in_drain_state(px_t *px_p) 2138 { 2139 return (B_FALSE); 2140 } 2141 2142 /* 2143 * There is no IOAPI to get the BDF of the pcie root port nexus at this moment. 2144 * Assume it is 0x0000, until otherwise noted. For now, all sun4v platforms 2145 * have programmed the BDF to be 0x0000. 2146 */ 2147 /*ARGSUSED*/ 2148 pcie_req_id_t 2149 px_lib_get_bdf(px_t *px_p) 2150 { 2151 return (0x0000); 2152 } 2153 2154 int 2155 px_lib_get_root_complex_mps(px_t *px_p, dev_info_t *dip, int *mps) 2156 { 2157 pci_device_t bdf = px_lib_get_bdf(px_p); 2158 2159 if (hvio_get_rp_mps_cap(DIP_TO_HANDLE(dip), bdf, mps) == H_EOK) 2160 return (DDI_SUCCESS); 2161 else 2162 return (DDI_FAILURE); 2163 } 2164 2165 int 2166 px_lib_set_root_complex_mps(px_t *px_p, dev_info_t *dip, int mps) 2167 { 2168 pci_device_t bdf = px_lib_get_bdf(px_p); 2169 2170 if (hvio_set_rp_mps(DIP_TO_HANDLE(dip), bdf, mps) == H_EOK) 2171 return (DDI_SUCCESS); 2172 else 2173 return (DDI_FAILURE); 2174 } 2175 2176 static int 2177 px_lib_do_count_waiting_dev(dev_info_t *dip, void *arg) 2178 { 2179 int *count = (int *)arg; 2180 dev_info_t *cdip = ddi_get_child(dip); 2181 2182 while (cdip != NULL) { 2183 /* check if this is an assigned device */ 2184 if (ddi_prop_exists(DDI_DEV_T_NONE, cdip, DDI_PROP_DONTPASS, 2185 "ddi-assigned")) { 2186 DBG(DBG_ATTACH, dip, "px_lib_do_count_waiting_dev: " 2187 "Found an assigned dev %p, under bridge %p", 2188 cdip, dip); 2189 2190 /* 2191 * Mark this bridge as needing waiting for 2192 * CHILD_LOANED will be removed after bridge reports 2193 * its readyness back to px driver 2194 */ 2195 if (ddi_prop_update_int(DDI_DEV_T_NONE, dip, 2196 CHILD_LOANED, 1) == DDI_PROP_SUCCESS) 2197 (*count)++; 2198 break; 2199 } 2200 cdip = ddi_get_next_sibling(cdip); 2201 } 2202 2203 return (DDI_WALK_CONTINUE); 2204 } 2205 2206 static int 2207 px_lib_count_waiting_dev(dev_info_t *dip) 2208 { 2209 int circular_count; 2210 int count = 0; 2211 2212 /* No need to continue if this system is not SDIO capable */ 2213 if (px_sdio_users == 0) 2214 return (0); 2215 2216 /* see if px iteslf has assigned children */ 2217 (void) px_lib_do_count_waiting_dev(dip, &count); 2218 2219 /* scan dev under this px */ 2220 ndi_devi_enter(dip, &circular_count); 2221 ddi_walk_devs(ddi_get_child(dip), px_lib_do_count_waiting_dev, &count); 2222 ndi_devi_exit(dip, circular_count); 2223 return (count); 2224 } 2225 2226 /* Called from px/bridge driver directly to report its readyness */ 2227 int 2228 px_lib_fabric_sync(dev_info_t *dip) 2229 { 2230 px_t *px; 2231 dev_info_t *rcdip; 2232 int waitdev; 2233 2234 /* No need to continue if this system is not SDIO capable */ 2235 if (px_sdio_users == 0) 2236 return (DDI_SUCCESS); 2237 2238 /* a valid bridge w/ assigned dev under it? */ 2239 if (ddi_prop_remove(DDI_DEV_T_NONE, dip, CHILD_LOANED) != 2240 DDI_PROP_SUCCESS) 2241 return (DDI_FAILURE); 2242 2243 /* find out RC dip */ 2244 for (rcdip = dip; rcdip != NULL; rcdip = ddi_get_parent(rcdip)) { 2245 if (PCIE_DIP2BUS(rcdip) && PCIE_IS_RC(PCIE_DIP2BUS(rcdip))) 2246 break; 2247 } 2248 if ((rcdip == NULL) || ((px = (px_t *)DIP_TO_STATE(rcdip)) == NULL)) 2249 return (DDI_FAILURE); 2250 2251 /* are we ready? */ 2252 waitdev = (int)(uintptr_t)px->px_plat_p; 2253 ASSERT(waitdev); 2254 DBG(DBG_CTLOPS, rcdip, "px_lib_fabric_sync: " 2255 "Px/bridge %p is ready, %d left", rcdip, waitdev - 1); 2256 --waitdev; 2257 px->px_plat_p = (void *)(uintptr_t)waitdev; 2258 if (waitdev != 0) 2259 return (DDI_SUCCESS); 2260 2261 /* notify hpyervisor */ 2262 DBG(DBG_CTLOPS, rcdip, "px_lib_fabric_sync: " 2263 "Notifying HV that RC %p is ready users=%d", rcdip, px_sdio_users); 2264 2265 if (pci_iov_root_configured(px->px_dev_hdl) != H_EOK) 2266 return (DDI_FAILURE); 2267 2268 return (DDI_SUCCESS); 2269 } 2270